diff --git a/runtime/kubernetes/kubernetes.go b/runtime/kubernetes/kubernetes.go index 7eca37ce..c69cf758 100644 --- a/runtime/kubernetes/kubernetes.go +++ b/runtime/kubernetes/kubernetes.go @@ -11,6 +11,7 @@ import ( "github.com/micro/go-micro/v3/logger" log "github.com/micro/go-micro/v3/logger" "github.com/micro/go-micro/v3/runtime" + "github.com/micro/go-micro/v3/util/kubernetes/api" "github.com/micro/go-micro/v3/util/kubernetes/client" ) @@ -218,7 +219,7 @@ func (k *kubernetes) getService(labels map[string]string, opts ...client.GetOpti // set status from waiting if v := state.Waiting; v != nil { - status = runtime.Pending + status = runtime.Starting } svc.Status(status, nil) @@ -577,7 +578,13 @@ func (k *kubernetes) Delete(s *runtime.Service, opts ...runtime.DeleteOption) er ns := client.DeleteNamespace(options.Namespace) k.client.Delete(&client.Resource{Name: credentialsName(s), Kind: "secret"}, ns) - return service.Stop(k.client, ns) + if err := service.Stop(k.client, ns); err == api.ErrNotFound { + return runtime.ErrNotFound + } else if err != nil { + return err + } + + return nil } // Start starts the runtime @@ -741,7 +748,7 @@ func (k *kubernetes) DeleteNamespace(ns string) error { func transformStatus(depStatus string) runtime.ServiceStatus { switch strings.ToLower(depStatus) { case "pending": - return runtime.Pending + return runtime.Starting case "containercreating": return runtime.Starting case "imagepullbackoff": @@ -759,7 +766,7 @@ func transformStatus(depStatus string) runtime.ServiceStatus { case "failed": return runtime.Error case "waiting": - return runtime.Pending + return runtime.Starting case "terminated": return runtime.Stopped default: diff --git a/runtime/local/local.go b/runtime/local/local.go index fd2c3ef7..ad24f091 100644 --- a/runtime/local/local.go +++ b/runtime/local/local.go @@ -14,7 +14,6 @@ import ( "github.com/hpcloud/tail" "github.com/micro/go-micro/v3/logger" "github.com/micro/go-micro/v3/runtime" - "github.com/micro/go-micro/v3/runtime/local/source/git" ) // defaultNamespace to use if not provided as an option @@ -63,68 +62,6 @@ func NewRuntime(opts ...runtime.Option) runtime.Runtime { } } -func (r *localRuntime) checkoutSourceIfNeeded(s *runtime.Service, secrets map[string]string) error { - // Runtime service like config have no source. - // Skip checkout in that case - if len(s.Source) == 0 { - return nil - } - - // Incoming uploaded files have format lastfolder.tar.gz or - // lastfolder.tar.gz/relative/path - sourceParts := strings.Split(s.Source, "/") - compressedFilepath := filepath.Join(SourceDir, sourceParts[0]) - uncompressPath := strings.ReplaceAll(compressedFilepath, ".tar.gz", "") - tarName := strings.ReplaceAll(sourceParts[0], ".tar.gz", "") - if len(sourceParts) > 1 { - uncompressPath = filepath.Join(SourceDir, tarName) - } - - // check if the directory already exists - if ex, _ := exists(compressedFilepath); ex { - err := os.RemoveAll(uncompressPath) - if err != nil { - return err - } - err = os.MkdirAll(uncompressPath, 0777) - if err != nil { - return err - } - err = git.Uncompress(compressedFilepath, uncompressPath) - if err != nil { - return err - } - if len(sourceParts) > 1 { - lastFolderPart := tarName - fullp := append([]string{uncompressPath}, sourceParts[1:]...) - s.Source = filepath.Join(append(fullp, lastFolderPart)...) - } else { - // The tar name is 'helloworld' for both - // the case when the code is uploaded from `$REPO/helloworld` - // and when it's uploaded from outside a repo ie `~/helloworld`. - if _, err := Entrypoint(filepath.Join(uncompressPath, tarName)); err == nil { - s.Source = filepath.Join(uncompressPath, tarName) - } else { - s.Source = uncompressPath - } - } - return nil - } - - source, err := git.ParseSourceLocal("", s.Source) - if err != nil { - return err - } - source.Ref = s.Version - - err = git.CheckoutSource(os.TempDir(), source, secrets) - if err != nil { - return err - } - s.Source = source.FullPath - return nil -} - // Init initializes runtime options func (r *localRuntime) Init(opts ...runtime.Option) error { r.Lock() @@ -284,16 +221,15 @@ func (r *localRuntime) Create(s *runtime.Service, opts ...runtime.CreateOption) o(&options) } - err := r.checkoutSourceIfNeeded(s, options.Secrets) - if err != nil { - return err - } r.Lock() defer r.Unlock() if len(options.Namespace) == 0 { options.Namespace = defaultNamespace } + if len(options.Entrypoint) > 0 { + s.Source = filepath.Join(s.Source, options.Entrypoint) + } if len(options.Command) == 0 { ep, err := Entrypoint(s.Source) if err != nil { @@ -501,14 +437,12 @@ func (r *localRuntime) Update(s *runtime.Service, opts ...runtime.UpdateOption) for _, o := range opts { o(&options) } - err := r.checkoutSourceIfNeeded(s, options.Secrets) - if err != nil { - return err - } - if len(options.Namespace) == 0 { options.Namespace = defaultNamespace } + if len(options.Entrypoint) > 0 { + s.Source = filepath.Join(s.Source, options.Entrypoint) + } r.Lock() srvs, ok := r.namespaces[options.Namespace] @@ -529,6 +463,9 @@ func (r *localRuntime) Update(s *runtime.Service, opts ...runtime.UpdateOption) return err } + // update the source to the new location and restart the service + service.Source = s.Source + service.Exec.Dir = s.Source return service.Start() } diff --git a/runtime/local/source/git/git.go b/runtime/local/source/git/git.go index 08094f09..c15eb6dc 100644 --- a/runtime/local/source/git/git.go +++ b/runtime/local/source/git/git.go @@ -71,7 +71,7 @@ func (g *binaryGitter) checkoutAnyRemote(repo, branchOrCommit string, useCredent } // Assumes remote address format is git@gitlab.com:micro-test/monorepo-test.git - remoteAddr := fmt.Sprintf("https://%v", repo) + remoteAddr := fmt.Sprintf("https://%v", strings.TrimPrefix(repo, "https://")) if useCredentials { remoteAddr = fmt.Sprintf("https://%v@%v", g.secrets[credentialsKey], repo) } @@ -264,9 +264,13 @@ func (g *binaryGitter) RepoDir() string { return g.folder } -func NewGitter(folder string, secrets map[string]string) Gitter { - return &binaryGitter{folder, secrets} +func NewGitter(secrets map[string]string) Gitter { + tmpdir, _ := ioutil.TempDir(os.TempDir(), "git-src-*") + return &binaryGitter{ + folder: tmpdir, + secrets: secrets, + } } func commandExists(cmd string) bool { @@ -348,6 +352,10 @@ func (s *Source) RuntimeName() string { // Source to be passed to RPC call runtime.Create Update Delete // eg: `helloworld`, `github.com/crufter/myrepo/helloworld`, `/path/to/localrepo/localfolder` func (s *Source) RuntimeSource() string { + if s.Local && s.LocalRepoRoot != s.FullPath { + relpath, _ := filepath.Rel(s.LocalRepoRoot, s.FullPath) + return relpath + } if s.Local { return s.FullPath } @@ -366,7 +374,13 @@ func ParseSource(source string) (*Source, error) { refs := strings.Split(source, "@") ret.Ref = refs[1] parts := strings.Split(refs[0], "/") - ret.Repo = strings.Join(parts[0:3], "/") + + max := 3 + if len(parts) < 3 { + max = len(parts) + } + ret.Repo = strings.Join(parts[0:max], "/") + if len(parts) > 1 { ret.Folder = strings.Join(parts[3:], "/") } @@ -430,25 +444,19 @@ func IsLocal(workDir, source string, pathExistsFunc ...func(path string) (bool, return false, "" } -// CheckoutSource for the local runtime server -// folder is the folder to check out the source code to -// Modifies source path to set it to checked out repo absolute path locally. -func CheckoutSource(folder string, source *Source, secrets map[string]string) error { - // if it's a local folder, do nothing - if exists, err := pathExists(source.FullPath); err == nil && exists { - return nil - } - gitter := NewGitter(folder, secrets) +// CheckoutSource checks out a git repo (source) into a local temp directory. It will reutrn the +// source of the local repo an an error if one occured. Secrets can optionally be passed if the repo +// is private. +func CheckoutSource(source *Source, secrets map[string]string) (string, error) { + gitter := NewGitter(secrets) repo := source.Repo if !strings.Contains(repo, "https://") { repo = "https://" + repo } - err := gitter.Checkout(source.Repo, source.Ref) - if err != nil { - return err + if err := gitter.Checkout(repo, source.Ref); err != nil { + return "", err } - source.FullPath = filepath.Join(gitter.RepoDir(), source.Folder) - return nil + return gitter.RepoDir(), nil } // code below is not used yet diff --git a/runtime/options.go b/runtime/options.go index 22c6e65a..661565f9 100644 --- a/runtime/options.go +++ b/runtime/options.go @@ -11,16 +11,16 @@ type Option func(o *Options) // Options configure runtime type Options struct { - // Scheduler for updates - Scheduler Scheduler - // Service type to manage - Type string - // Source of the services repository - Source string - // Base image to use - Image string // Client to use when making requests Client client.Client + // Base image to use + Image string + // Scheduler for updates + Scheduler Scheduler + // Source of the services repository + Source string + // Service type to manage + Type string } // WithSource sets the base image / repository @@ -70,6 +70,8 @@ type CreateOptions struct { Args []string // Environment to configure Env []string + // Entrypoint within the folder (e.g. in the case of a mono-repo) + Entrypoint string // Log output Output io.Writer // Type of service to create @@ -132,6 +134,13 @@ func CreateContext(ctx context.Context) CreateOption { } } +// CreateEntrypoint sets the entrypoint +func CreateEntrypoint(e string) CreateOption { + return func(o *CreateOptions) { + o.Entrypoint = e + } +} + // WithSecret sets a secret to provide the service with func WithSecret(key, value string) CreateOption { return func(o *CreateOptions) { @@ -236,6 +245,8 @@ func ReadContext(ctx context.Context) ReadOption { type UpdateOption func(o *UpdateOptions) type UpdateOptions struct { + // Entrypoint within the folder (e.g. in the case of a mono-repo) + Entrypoint string // Namespace the service is running in Namespace string // Specify the context to use @@ -269,6 +280,13 @@ func UpdateContext(ctx context.Context) UpdateOption { } } +// UpdateEntrypoint sets the entrypoint +func UpdateEntrypoint(e string) UpdateOption { + return func(o *UpdateOptions) { + o.Entrypoint = e + } +} + type DeleteOption func(o *DeleteOptions) type DeleteOptions struct { diff --git a/store/s3/options.go b/store/s3/options.go index 266bee90..534ef2d6 100644 --- a/store/s3/options.go +++ b/store/s3/options.go @@ -1,5 +1,7 @@ package s3 +import "crypto/tls" + // Options used to configure the s3 blob store type Options struct { Endpoint string @@ -7,6 +9,7 @@ type Options struct { AccessKeyID string SecretAccessKey string Secure bool + TLSConfig *tls.Config } // Option configures one or more options @@ -40,3 +43,10 @@ func Insecure() Option { o.Secure = false } } + +// TLSConfig sets the tls config for the client +func TLSConfig(c *tls.Config) Option { + return func(o *Options) { + o.TLSConfig = c + } +} diff --git a/store/s3/s3.go b/store/s3/s3.go index 52fd69bf..73cca10b 100644 --- a/store/s3/s3.go +++ b/store/s3/s3.go @@ -6,6 +6,7 @@ import ( "io" "io/ioutil" "net/http" + "regexp" "github.com/micro/go-micro/v3/store" "github.com/minio/minio-go/v7" @@ -13,6 +14,8 @@ import ( "github.com/pkg/errors" ) +var keyRegex = regexp.MustCompile("[^a-zA-Z0-9]+") + // NewBlobStore returns an initialized s3 blob store func NewBlobStore(opts ...Option) (store.BlobStore, error) { // parse the options @@ -20,12 +23,25 @@ func NewBlobStore(opts ...Option) (store.BlobStore, error) { for _, o := range opts { o(&options) } + minioOpts := &minio.Options{ + Secure: options.Secure, + } + if len(options.AccessKeyID) > 0 || len(options.SecretAccessKey) > 0 { + minioOpts.Creds = credentials.NewStaticV4(options.AccessKeyID, options.SecretAccessKey, "") + } + + // configure the transport to use custom tls config if provided + if options.TLSConfig != nil { + ts, err := minio.DefaultTransport(options.Secure) + if err != nil { + return nil, errors.Wrap(err, "Error setting up s3 blob store transport") + } + ts.TLSClientConfig = options.TLSConfig + minioOpts.Transport = ts + } // initialize minio client - client, err := minio.New(options.Endpoint, &minio.Options{ - Creds: credentials.NewStaticV4(options.AccessKeyID, options.SecretAccessKey, ""), - Secure: options.Secure, - }) + client, err := minio.New(options.Endpoint, minioOpts) if err != nil { return nil, errors.Wrap(err, "Error connecting to s3 blob store") } @@ -45,6 +61,9 @@ func (s *s3) Read(key string, opts ...store.BlobOption) (io.Reader, error) { return nil, store.ErrMissingKey } + // make the key safe for use with s3 + key = keyRegex.ReplaceAllString(key, "-") + // parse the options var options store.BlobOptions for _, o := range opts { @@ -87,6 +106,9 @@ func (s *s3) Write(key string, blob io.Reader, opts ...store.BlobOption) error { return store.ErrMissingKey } + // make the key safe for use with s3 + key = keyRegex.ReplaceAllString(key, "-") + // parse the options var options store.BlobOptions for _, o := range opts { @@ -130,6 +152,9 @@ func (s *s3) Delete(key string, opts ...store.BlobOption) error { return store.ErrMissingKey } + // make the key safe for use with s3 + key = keyRegex.ReplaceAllString(key, "-") + // parse the options var options store.BlobOptions for _, o := range opts {