Fixes for runtime builder (#2029)

* util/tar: add archive util funcs

* runtime: add store, builder options

* runtime/local: update RuntimeSource func

* runtime/builder/golang: use tar util

* store/s3: make keys safe

* runtime: add entrypoint options

* runtime/builder: remove debugging

* wip: integrate builder into k8s runtime

* runtime/builder/golang: build for a linux architecture

* runtime/kubernetes: write builds to the users namespace

* runtime/local/source: fixes for mono-repo builds

* runtime/local: stop checking out source (the responsibility is on the client)

* runtime/builder: fix golang tests

* runtime/local: fix out of bounds panic

* Update

* revert changes

* runtime/local/source: refactor git package (wip)

* runtime/kubernetes: map err not found

* fix TestRunGenericRemote test

* runtime/local: fix update not reassining source

* Tidy go mod

* runtime.Pending => runtime.Starting

* store/s3: only use credentials option when set

* store/s3: add tls config option

runtime/kubernetes/kubernetes.go

@@ -11,6 +11,7 @@ import (
 	"github.com/micro/go-micro/v3/logger"
 	log "github.com/micro/go-micro/v3/logger"
 	"github.com/micro/go-micro/v3/runtime"
+	"github.com/micro/go-micro/v3/util/kubernetes/api"
 	"github.com/micro/go-micro/v3/util/kubernetes/client"
 )
 
@@ -218,7 +219,7 @@ func (k *kubernetes) getService(labels map[string]string, opts ...client.GetOpti
 
 				// set status from waiting
 				if v := state.Waiting; v != nil {
-					status = runtime.Pending
+					status = runtime.Starting
 				}
 
 				svc.Status(status, nil)
@@ -577,7 +578,13 @@ func (k *kubernetes) Delete(s *runtime.Service, opts ...runtime.DeleteOption) er
 	ns := client.DeleteNamespace(options.Namespace)
 	k.client.Delete(&client.Resource{Name: credentialsName(s), Kind: "secret"}, ns)
 
-	return service.Stop(k.client, ns)
+	if err := service.Stop(k.client, ns); err == api.ErrNotFound {
+		return runtime.ErrNotFound
+	} else if err != nil {
+		return err
+	}
+
+	return nil
 }
 
 // Start starts the runtime
@@ -741,7 +748,7 @@ func (k *kubernetes) DeleteNamespace(ns string) error {
 func transformStatus(depStatus string) runtime.ServiceStatus {
 	switch strings.ToLower(depStatus) {
 	case "pending":
-		return runtime.Pending
+		return runtime.Starting
 	case "containercreating":
 		return runtime.Starting
 	case "imagepullbackoff":
@@ -759,7 +766,7 @@ func transformStatus(depStatus string) runtime.ServiceStatus {
 	case "failed":
 		return runtime.Error
 	case "waiting":
-		return runtime.Pending
+		return runtime.Starting
 	case "terminated":
 		return runtime.Stopped
 	default:

runtime/local/local.go

@@ -14,7 +14,6 @@ import (
 	"github.com/hpcloud/tail"
 	"github.com/micro/go-micro/v3/logger"
 	"github.com/micro/go-micro/v3/runtime"
-	"github.com/micro/go-micro/v3/runtime/local/source/git"
 )
 
 // defaultNamespace to use if not provided as an option
@@ -63,68 +62,6 @@ func NewRuntime(opts ...runtime.Option) runtime.Runtime {
 	}
 }
 
-func (r *localRuntime) checkoutSourceIfNeeded(s *runtime.Service, secrets map[string]string) error {
-	// Runtime service like config have no source.
-	// Skip checkout in that case
-	if len(s.Source) == 0 {
-		return nil
-	}
-
-	// Incoming uploaded files have format lastfolder.tar.gz or
-	// lastfolder.tar.gz/relative/path
-	sourceParts := strings.Split(s.Source, "/")
-	compressedFilepath := filepath.Join(SourceDir, sourceParts[0])
-	uncompressPath := strings.ReplaceAll(compressedFilepath, ".tar.gz", "")
-	tarName := strings.ReplaceAll(sourceParts[0], ".tar.gz", "")
-	if len(sourceParts) > 1 {
-		uncompressPath = filepath.Join(SourceDir, tarName)
-	}
-
-	// check if the directory already exists
-	if ex, _ := exists(compressedFilepath); ex {
-		err := os.RemoveAll(uncompressPath)
-		if err != nil {
-			return err
-		}
-		err = os.MkdirAll(uncompressPath, 0777)
-		if err != nil {
-			return err
-		}
-		err = git.Uncompress(compressedFilepath, uncompressPath)
-		if err != nil {
-			return err
-		}
-		if len(sourceParts) > 1 {
-			lastFolderPart := tarName
-			fullp := append([]string{uncompressPath}, sourceParts[1:]...)
-			s.Source = filepath.Join(append(fullp, lastFolderPart)...)
-		} else {
-			// The tar name is 'helloworld' for both
-			// the case when the code is uploaded from `$REPO/helloworld`
-			// and when it's uploaded from outside a repo ie `~/helloworld`.
-			if _, err := Entrypoint(filepath.Join(uncompressPath, tarName)); err == nil {
-				s.Source = filepath.Join(uncompressPath, tarName)
-			} else {
-				s.Source = uncompressPath
-			}
-		}
-		return nil
-	}
-
-	source, err := git.ParseSourceLocal("", s.Source)
-	if err != nil {
-		return err
-	}
-	source.Ref = s.Version
-
-	err = git.CheckoutSource(os.TempDir(), source, secrets)
-	if err != nil {
-		return err
-	}
-	s.Source = source.FullPath
-	return nil
-}
-
 // Init initializes runtime options
 func (r *localRuntime) Init(opts ...runtime.Option) error {
 	r.Lock()
@@ -284,16 +221,15 @@ func (r *localRuntime) Create(s *runtime.Service, opts ...runtime.CreateOption)
 		o(&options)
 	}
 
-	err := r.checkoutSourceIfNeeded(s, options.Secrets)
-	if err != nil {
-		return err
-	}
 	r.Lock()
 	defer r.Unlock()
 
 	if len(options.Namespace) == 0 {
 		options.Namespace = defaultNamespace
 	}
+	if len(options.Entrypoint) > 0 {
+		s.Source = filepath.Join(s.Source, options.Entrypoint)
+	}
 	if len(options.Command) == 0 {
 		ep, err := Entrypoint(s.Source)
 		if err != nil {
@@ -501,14 +437,12 @@ func (r *localRuntime) Update(s *runtime.Service, opts ...runtime.UpdateOption)
 	for _, o := range opts {
 		o(&options)
 	}
-	err := r.checkoutSourceIfNeeded(s, options.Secrets)
-	if err != nil {
-		return err
-	}
-
 	if len(options.Namespace) == 0 {
 		options.Namespace = defaultNamespace
 	}
+	if len(options.Entrypoint) > 0 {
+		s.Source = filepath.Join(s.Source, options.Entrypoint)
+	}
 
 	r.Lock()
 	srvs, ok := r.namespaces[options.Namespace]
@@ -529,6 +463,9 @@ func (r *localRuntime) Update(s *runtime.Service, opts ...runtime.UpdateOption)
 		return err
 	}
 
+	// update the source to the new location and restart the service
+	service.Source = s.Source
+	service.Exec.Dir = s.Source
 	return service.Start()
 }
 

runtime/local/source/git/git.go

@@ -71,7 +71,7 @@ func (g *binaryGitter) checkoutAnyRemote(repo, branchOrCommit string, useCredent
 	}
 
 	// Assumes remote address format is git@gitlab.com:micro-test/monorepo-test.git
-	remoteAddr := fmt.Sprintf("https://%v", repo)
+	remoteAddr := fmt.Sprintf("https://%v", strings.TrimPrefix(repo, "https://"))
 	if useCredentials {
 		remoteAddr = fmt.Sprintf("https://%v@%v", g.secrets[credentialsKey], repo)
 	}
@@ -264,9 +264,13 @@ func (g *binaryGitter) RepoDir() string {
 	return g.folder
 }
 
-func NewGitter(folder string, secrets map[string]string) Gitter {
+func NewGitter(secrets map[string]string) Gitter {
-	return &binaryGitter{folder, secrets}
+	tmpdir, _ := ioutil.TempDir(os.TempDir(), "git-src-*")
 
+	return &binaryGitter{
+		folder:  tmpdir,
+		secrets: secrets,
+	}
 }
 
 func commandExists(cmd string) bool {
@@ -348,6 +352,10 @@ func (s *Source) RuntimeName() string {
 // Source to be passed to RPC call runtime.Create Update Delete
 // eg: `helloworld`, `github.com/crufter/myrepo/helloworld`, `/path/to/localrepo/localfolder`
 func (s *Source) RuntimeSource() string {
+	if s.Local && s.LocalRepoRoot != s.FullPath {
+		relpath, _ := filepath.Rel(s.LocalRepoRoot, s.FullPath)
+		return relpath
+	}
 	if s.Local {
 		return s.FullPath
 	}
@@ -366,7 +374,13 @@ func ParseSource(source string) (*Source, error) {
 	refs := strings.Split(source, "@")
 	ret.Ref = refs[1]
 	parts := strings.Split(refs[0], "/")
-	ret.Repo = strings.Join(parts[0:3], "/")
+
+	max := 3
+	if len(parts) < 3 {
+		max = len(parts)
+	}
+	ret.Repo = strings.Join(parts[0:max], "/")
+
 	if len(parts) > 1 {
 		ret.Folder = strings.Join(parts[3:], "/")
 	}
@@ -430,25 +444,19 @@ func IsLocal(workDir, source string, pathExistsFunc ...func(path string) (bool,
 	return false, ""
 }
 
-// CheckoutSource for the local runtime server
+// CheckoutSource checks out a git repo (source) into a local temp directory. It will reutrn the
-// folder is the folder to check out the source code to
+// source of the local repo an an error if one occured. Secrets can optionally be passed if the repo
-// Modifies source path to set it to checked out repo absolute path locally.
+// is private.
-func CheckoutSource(folder string, source *Source, secrets map[string]string) error {
+func CheckoutSource(source *Source, secrets map[string]string) (string, error) {
-	// if it's a local folder, do nothing
+	gitter := NewGitter(secrets)
-	if exists, err := pathExists(source.FullPath); err == nil && exists {
-		return nil
-	}
-	gitter := NewGitter(folder, secrets)
 	repo := source.Repo
 	if !strings.Contains(repo, "https://") {
 		repo = "https://" + repo
 	}
-	err := gitter.Checkout(source.Repo, source.Ref)
+	if err := gitter.Checkout(repo, source.Ref); err != nil {
-	if err != nil {
+		return "", err
-		return err
 	}
-	source.FullPath = filepath.Join(gitter.RepoDir(), source.Folder)
+	return gitter.RepoDir(), nil
-	return nil
 }
 
 // code below is not used yet

runtime/options.go

@@ -11,16 +11,16 @@ type Option func(o *Options)
 
 // Options configure runtime
 type Options struct {
-	// Scheduler for updates
-	Scheduler Scheduler
-	// Service type to manage
-	Type string
-	// Source of the services repository
-	Source string
-	// Base image to use
-	Image string
 	// Client to use when making requests
 	Client client.Client
+	// Base image to use
+	Image string
+	// Scheduler for updates
+	Scheduler Scheduler
+	// Source of the services repository
+	Source string
+	// Service type to manage
+	Type string
 }
 
 // WithSource sets the base image / repository
@@ -70,6 +70,8 @@ type CreateOptions struct {
 	Args []string
 	// Environment to configure
 	Env []string
+	// Entrypoint within the folder (e.g. in the case of a mono-repo)
+	Entrypoint string
 	// Log output
 	Output io.Writer
 	// Type of service to create
@@ -132,6 +134,13 @@ func CreateContext(ctx context.Context) CreateOption {
 	}
 }
 
+// CreateEntrypoint sets the entrypoint
+func CreateEntrypoint(e string) CreateOption {
+	return func(o *CreateOptions) {
+		o.Entrypoint = e
+	}
+}
+
 // WithSecret sets a secret to provide the service with
 func WithSecret(key, value string) CreateOption {
 	return func(o *CreateOptions) {
@@ -236,6 +245,8 @@ func ReadContext(ctx context.Context) ReadOption {
 type UpdateOption func(o *UpdateOptions)
 
 type UpdateOptions struct {
+	// Entrypoint within the folder (e.g. in the case of a mono-repo)
+	Entrypoint string
 	// Namespace the service is running in
 	Namespace string
 	// Specify the context to use
@@ -269,6 +280,13 @@ func UpdateContext(ctx context.Context) UpdateOption {
 	}
 }
 
+// UpdateEntrypoint sets the entrypoint
+func UpdateEntrypoint(e string) UpdateOption {
+	return func(o *UpdateOptions) {
+		o.Entrypoint = e
+	}
+}
+
 type DeleteOption func(o *DeleteOptions)
 
 type DeleteOptions struct {

store/s3/options.go

@@ -1,5 +1,7 @@
 package s3
 
+import "crypto/tls"
+
 // Options used to configure the s3 blob store
 type Options struct {
 	Endpoint        string
@@ -7,6 +9,7 @@ type Options struct {
 	AccessKeyID     string
 	SecretAccessKey string
 	Secure          bool
+	TLSConfig       *tls.Config
 }
 
 // Option configures one or more options
@@ -40,3 +43,10 @@ func Insecure() Option {
 		o.Secure = false
 	}
 }
+
+// TLSConfig sets the tls config for the client
+func TLSConfig(c *tls.Config) Option {
+	return func(o *Options) {
+		o.TLSConfig = c
+	}
+}

store/s3/s3.go

@@ -6,6 +6,7 @@ import (
 	"io"
 	"io/ioutil"
 	"net/http"
+	"regexp"
 
 	"github.com/micro/go-micro/v3/store"
 	"github.com/minio/minio-go/v7"
@@ -13,6 +14,8 @@ import (
 	"github.com/pkg/errors"
 )
 
+var keyRegex = regexp.MustCompile("[^a-zA-Z0-9]+")
+
 // NewBlobStore returns an initialized s3 blob store
 func NewBlobStore(opts ...Option) (store.BlobStore, error) {
 	// parse the options
@@ -20,12 +23,25 @@ func NewBlobStore(opts ...Option) (store.BlobStore, error) {
 	for _, o := range opts {
 		o(&options)
 	}
+	minioOpts := &minio.Options{
+		Secure: options.Secure,
+	}
+	if len(options.AccessKeyID) > 0 || len(options.SecretAccessKey) > 0 {
+		minioOpts.Creds = credentials.NewStaticV4(options.AccessKeyID, options.SecretAccessKey, "")
+	}
+
+	// configure the transport to use custom tls config if provided
+	if options.TLSConfig != nil {
+		ts, err := minio.DefaultTransport(options.Secure)
+		if err != nil {
+			return nil, errors.Wrap(err, "Error setting up s3 blob store transport")
+		}
+		ts.TLSClientConfig = options.TLSConfig
+		minioOpts.Transport = ts
+	}
 
 	// initialize minio client
-	client, err := minio.New(options.Endpoint, &minio.Options{
+	client, err := minio.New(options.Endpoint, minioOpts)
-		Creds:  credentials.NewStaticV4(options.AccessKeyID, options.SecretAccessKey, ""),
-		Secure: options.Secure,
-	})
 	if err != nil {
 		return nil, errors.Wrap(err, "Error connecting to s3 blob store")
 	}
@@ -45,6 +61,9 @@ func (s *s3) Read(key string, opts ...store.BlobOption) (io.Reader, error) {
 		return nil, store.ErrMissingKey
 	}
 
+	// make the key safe for use with s3
+	key = keyRegex.ReplaceAllString(key, "-")
+
 	// parse the options
 	var options store.BlobOptions
 	for _, o := range opts {
@@ -87,6 +106,9 @@ func (s *s3) Write(key string, blob io.Reader, opts ...store.BlobOption) error {
 		return store.ErrMissingKey
 	}
 
+	// make the key safe for use with s3
+	key = keyRegex.ReplaceAllString(key, "-")
+
 	// parse the options
 	var options store.BlobOptions
 	for _, o := range opts {
@@ -130,6 +152,9 @@ func (s *s3) Delete(key string, opts ...store.BlobOption) error {
 		return store.ErrMissingKey
 	}
 
+	// make the key safe for use with s3
+	key = keyRegex.ReplaceAllString(key, "-")
+
 	// parse the options
 	var options store.BlobOptions
 	for _, o := range opts {