unistack-org/micro
4
0
Fork 2
Code Issues 18 Pull Requests Actions Packages Projects Releases 324 Wiki Activity

Fixes for runtime builder (#2029)

Browse Source 
* util/tar: add archive util funcs

* runtime: add store, builder options

* runtime/local: update RuntimeSource func

* runtime/builder/golang: use tar util

* store/s3: make keys safe

* runtime: add entrypoint options

* runtime/builder: remove debugging

* wip: integrate builder into k8s runtime

* runtime/builder/golang: build for a linux architecture

* runtime/kubernetes: write builds to the users namespace

* runtime/local/source: fixes for mono-repo builds

* runtime/local: stop checking out source (the responsibility is on the client)

* runtime/builder: fix golang tests

* runtime/local: fix out of bounds panic

* Update

* revert changes

* runtime/local/source: refactor git package (wip)

* runtime/kubernetes: map err not found

* fix TestRunGenericRemote test

* runtime/local: fix update not reassining source

* Tidy go mod

* runtime.Pending => runtime.Starting

* store/s3: only use credentials option when set

* store/s3: add tls config option
This commit is contained in:
ben-toogood
2020-10-02 09:54:26 +01:00
committed by GitHub
parent 231cfc48f0
commit 0a6e451539
6 changed files with 111 additions and 106 deletions
Download Patch File Download Diff File Expand all files Collapse all files

81
runtime/local/local.go
View File

@@ -14,7 +14,6 @@ import (
"github.com/hpcloud/tail"
"github.com/micro/go-micro/v3/logger"
"github.com/micro/go-micro/v3/runtime"
"github.com/micro/go-micro/v3/runtime/local/source/git"
)
// defaultNamespace to use if not provided as an option
@@ -63,68 +62,6 @@ func NewRuntime(opts ...runtime.Option) runtime.Runtime {
}
}
func (r *localRuntime) checkoutSourceIfNeeded(s *runtime.Service, secrets map[string]string) error {
// Runtime service like config have no source.
// Skip checkout in that case
if len(s.Source) == 0 {
return nil
}
// Incoming uploaded files have format lastfolder.tar.gz or
// lastfolder.tar.gz/relative/path
sourceParts := strings.Split(s.Source, "/")
compressedFilepath := filepath.Join(SourceDir, sourceParts[0])
uncompressPath := strings.ReplaceAll(compressedFilepath, ".tar.gz", "")
tarName := strings.ReplaceAll(sourceParts[0], ".tar.gz", "")
if len(sourceParts) > 1 {
uncompressPath = filepath.Join(SourceDir, tarName)
}
// check if the directory already exists
if ex, _ := exists(compressedFilepath); ex {
err := os.RemoveAll(uncompressPath)
if err != nil {
return err
}
err = os.MkdirAll(uncompressPath, 0777)
if err != nil {
return err
}
err = git.Uncompress(compressedFilepath, uncompressPath)
if err != nil {
return err
}
if len(sourceParts) > 1 {
lastFolderPart := tarName
fullp := append([]string{uncompressPath}, sourceParts[1:]...)
s.Source = filepath.Join(append(fullp, lastFolderPart)...)
} else {
// The tar name is 'helloworld' for both
// the case when the code is uploaded from `$REPO/helloworld`
// and when it's uploaded from outside a repo ie `~/helloworld`.
if _, err := Entrypoint(filepath.Join(uncompressPath, tarName)); err == nil {
s.Source = filepath.Join(uncompressPath, tarName)
} else {
s.Source = uncompressPath
}
}
return nil
}
source, err := git.ParseSourceLocal("", s.Source)
if err != nil {
return err
}
source.Ref = s.Version
err = git.CheckoutSource(os.TempDir(), source, secrets)
if err != nil {
return err
}
s.Source = source.FullPath
return nil
}
// Init initializes runtime options
func (r *localRuntime) Init(opts ...runtime.Option) error {
r.Lock()
@@ -284,16 +221,15 @@ func (r *localRuntime) Create(s *runtime.Service, opts ...runtime.CreateOption)
o(&options)
}
err := r.checkoutSourceIfNeeded(s, options.Secrets)
if err != nil {
return err
}
r.Lock()
defer r.Unlock()
if len(options.Namespace) == 0 {
options.Namespace = defaultNamespace
}
if len(options.Entrypoint) > 0 {
s.Source = filepath.Join(s.Source, options.Entrypoint)
}
if len(options.Command) == 0 {
ep, err := Entrypoint(s.Source)
if err != nil {
@@ -501,14 +437,12 @@ func (r *localRuntime) Update(s *runtime.Service, opts ...runtime.UpdateOption)
for _, o := range opts {
o(&options)
}
err := r.checkoutSourceIfNeeded(s, options.Secrets)
if err != nil {
return err
}
if len(options.Namespace) == 0 {
options.Namespace = defaultNamespace
}
if len(options.Entrypoint) > 0 {
s.Source = filepath.Join(s.Source, options.Entrypoint)
}
r.Lock()
srvs, ok := r.namespaces[options.Namespace]
@@ -529,6 +463,9 @@ func (r *localRuntime) Update(s *runtime.Service, opts ...runtime.UpdateOption)
return err
}
// update the source to the new location and restart the service
service.Source = s.Source
service.Exec.Dir = s.Source
return service.Start()
}

44
runtime/local/source/git/git.go
View File

@@ -71,7 +71,7 @@ func (g *binaryGitter) checkoutAnyRemote(repo, branchOrCommit string, useCredent
}
// Assumes remote address format is git@gitlab.com:micro-test/monorepo-test.git
remoteAddr := fmt.Sprintf("https://%v", repo)
remoteAddr := fmt.Sprintf("https://%v", strings.TrimPrefix(repo, "https://"))
if useCredentials {
remoteAddr = fmt.Sprintf("https://%v@%v", g.secrets[credentialsKey], repo)
}
@@ -264,9 +264,13 @@ func (g *binaryGitter) RepoDir() string {
return g.folder
}
func NewGitter(folder string, secrets map[string]string) Gitter {
return &binaryGitter{folder, secrets}
func NewGitter(secrets map[string]string) Gitter {
tmpdir, _ := ioutil.TempDir(os.TempDir(), "git-src-*")
return &binaryGitter{
folder: tmpdir,
secrets: secrets,
}
}
func commandExists(cmd string) bool {
@@ -348,6 +352,10 @@ func (s *Source) RuntimeName() string {
// Source to be passed to RPC call runtime.Create Update Delete
// eg: `helloworld`, `github.com/crufter/myrepo/helloworld`, `/path/to/localrepo/localfolder`
func (s *Source) RuntimeSource() string {
if s.Local && s.LocalRepoRoot != s.FullPath {
relpath, _ := filepath.Rel(s.LocalRepoRoot, s.FullPath)
return relpath
}
if s.Local {
return s.FullPath
}
@@ -366,7 +374,13 @@ func ParseSource(source string) (*Source, error) {
refs := strings.Split(source, "@")
ret.Ref = refs[1]
parts := strings.Split(refs[0], "/")
ret.Repo = strings.Join(parts[0:3], "/")
max := 3
if len(parts) < 3 {
max = len(parts)
}
ret.Repo = strings.Join(parts[0:max], "/")
if len(parts) > 1 {
ret.Folder = strings.Join(parts[3:], "/")
}
@@ -430,25 +444,19 @@ func IsLocal(workDir, source string, pathExistsFunc ...func(path string) (bool,
return false, ""
}
// CheckoutSource for the local runtime server
// folder is the folder to check out the source code to
// Modifies source path to set it to checked out repo absolute path locally.
func CheckoutSource(folder string, source *Source, secrets map[string]string) error {
// if it's a local folder, do nothing
if exists, err := pathExists(source.FullPath); err == nil && exists {
return nil
}
gitter := NewGitter(folder, secrets)
// CheckoutSource checks out a git repo (source) into a local temp directory. It will reutrn the
// source of the local repo an an error if one occured. Secrets can optionally be passed if the repo
// is private.
func CheckoutSource(source *Source, secrets map[string]string) (string, error) {
gitter := NewGitter(secrets)
repo := source.Repo
if !strings.Contains(repo, "https://") {
repo = "https://" + repo
}
err := gitter.Checkout(source.Repo, source.Ref)
if err != nil {
return err
if err := gitter.Checkout(repo, source.Ref); err != nil {
return "", err
}
source.FullPath = filepath.Join(gitter.RepoDir(), source.Folder)
return nil
return gitter.RepoDir(), nil
}
// code below is not used yet