From 2b2dc2f811d2f85f094a18f9d91806e9b7c605e8 Mon Sep 17 00:00:00 2001 From: Janos Dobronszki Date: Tue, 18 Aug 2020 18:26:14 +0200 Subject: [PATCH] Support private repos in env 'local' (#1938) --- go.mod | 2 +- go.sum | 7 + runtime/local/git/git.go | 259 +++++++++++++++++++++----------- runtime/local/local.go | 24 +-- runtime/local/source/git/git.go | 87 ----------- runtime/options.go | 13 ++ 6 files changed, 207 insertions(+), 185 deletions(-) delete mode 100644 runtime/local/source/git/git.go diff --git a/go.mod b/go.mod index 86ca5741..38b19cd9 100644 --- a/go.mod +++ b/go.mod @@ -56,12 +56,12 @@ require ( github.com/stretchr/testify v1.5.1 github.com/teris-io/shortid v0.0.0-20171029131806-771a37caa5cf github.com/tmc/grpc-websocket-proxy v0.0.0-20200122045848-3419fae592fc // indirect + github.com/xanzy/go-gitlab v0.35.1 // indirect github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect go.etcd.io/bbolt v1.3.5 go.uber.org/zap v1.13.0 golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899 golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2 - golang.org/x/time v0.0.0-20191024005414-555d28b269f0 // indirect golang.org/x/tools v0.0.0-20200117065230-39095c1d176c // indirect google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 google.golang.org/grpc v1.27.0 diff --git a/go.sum b/go.sum index a09f0eaa..5d5009d5 100644 --- a/go.sum +++ b/go.sum @@ -214,6 +214,7 @@ github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.0 h1:/QaMHBdZ26BB3SSst0Iwl10Epc+xhTquomWX0oZEB6w= github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= @@ -471,6 +472,8 @@ github.com/uber-go/atomic v1.3.2/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex github.com/urfave/cli v0.0.0-20171014202726-7bc6a0acffa5/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/vultr/govultr v0.1.4/go.mod h1:9H008Uxr/C4vFNGLqKx232C206GL0PBHzOP0809bGNA= +github.com/xanzy/go-gitlab v0.35.1 h1:jJSgT0NxjCvrSZf7Gvn2NxxV9xAYkTjYrKW8XwWhrfY= +github.com/xanzy/go-gitlab v0.35.1/go.mod h1:sPLojNBn68fMUWSxIJtdVVIP8uSBYqesTfDUseX11Ug= github.com/xanzy/ssh-agent v0.2.1 h1:TCbipTQL2JiiCprBWx9frJ2eJlCYT00NmctrHxVAr70= github.com/xanzy/ssh-agent v0.2.1/go.mod h1:mLlQY/MoOhWBj+gOGMQkOeiEvkx+8pJSI+0Bx9h2kr4= github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU= @@ -538,6 +541,7 @@ golang.org/x/net v0.0.0-20180611182652-db08ff08e862/go.mod h1:mL1N/T3taQHkDXs73r golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20181108082009-03003ca0c849/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -560,7 +564,9 @@ golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2 h1:eDrdRpKgkcCqKZQwyZRyeFZgfqt37SL7Kv3tok06cKE= golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= +golang.org/x/oauth2 v0.0.0-20181106182150-f42d05182288/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= +golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -645,6 +651,7 @@ google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEn google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg= google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= +google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= diff --git a/runtime/local/git/git.go b/runtime/local/git/git.go index 539e66c3..f2a80d0e 100644 --- a/runtime/local/git/git.go +++ b/runtime/local/git/git.go @@ -16,6 +16,7 @@ import ( "strings" "github.com/teris-io/shortid" + "github.com/xanzy/go-gitlab" ) type Gitter interface { @@ -24,7 +25,8 @@ type Gitter interface { } type binaryGitter struct { - folder string + folder string + secrets map[string]string } func (g *binaryGitter) Checkout(repo, branchOrCommit string) error { @@ -38,100 +40,187 @@ func (g *binaryGitter) Checkout(repo, branchOrCommit string) error { branchOrCommit = "master" } if strings.Contains(repo, "github") { - // @todo if it's a commit it must not be checked out all the time - repoFolder := strings.ReplaceAll(strings.ReplaceAll(repo, "/", "-"), "https://", "") - g.folder = filepath.Join(os.TempDir(), - repoFolder+"-"+shortid.MustGenerate()) - - url := fmt.Sprintf("%v/archive/%v.zip", repo, branchOrCommit) - if !strings.HasPrefix(url, "https://") { - url = "https://" + url - } - resp, err := http.Get(url) - if err != nil { - return fmt.Errorf("Can't get zip: %v", err) - } - - defer resp.Body.Close() - // Github returns 404 for tar.gz files... - // but still gives back a proper file so ignoring status code - // for now. - //if resp.StatusCode != 200 { - // return errors.New("Status code was not 200") - //} - - src := g.folder + ".zip" - // Create the file - out, err := os.Create(src) - if err != nil { - return fmt.Errorf("Can't create source file %v src: %v", src, err) - } - defer out.Close() - - // Write the body to file - _, err = io.Copy(out, resp.Body) - if err != nil { - return err - } - return unzip(src, g.folder, true) + return g.checkoutGithub(repo, branchOrCommit) } else if strings.Contains(repo, "gitlab") { - // Example: https://gitlab.com/micro-test/basic-micro-service/-/archive/master/basic-micro-service-master.tar.gz - // @todo if it's a commit it must not be checked out all the time - repoFolder := strings.ReplaceAll(strings.ReplaceAll(repo, "/", "-"), "https://", "") - g.folder = filepath.Join(os.TempDir(), - repoFolder+"-"+shortid.MustGenerate()) - - tarName := strings.ReplaceAll(strings.ReplaceAll(repo, "gitlab.com/", ""), "/", "-") - url := fmt.Sprintf("%v/-/archive/%v/%v.tar.gz", repo, branchOrCommit, tarName) - if !strings.HasPrefix(url, "https://") { - url = "https://" + url - } - resp, err := http.Get(url) + err := g.checkoutGitLabPublic(repo, branchOrCommit) if err != nil { - return fmt.Errorf("Can't get zip: %v", err) + // If the public download fails, try getting it with tokens. + // Private downloads needs a token for api project listing, hence + // the weird structure of this code. + return g.checkoutGitLabPrivate(repo, branchOrCommit) } - - defer resp.Body.Close() - - src := g.folder + ".tar.gz" - // Create the file - out, err := os.Create(src) - if err != nil { - return fmt.Errorf("Can't create source file %v src: %v", src, err) - } - defer out.Close() - - // Write the body to file - _, err = io.Copy(out, resp.Body) - if err != nil { - return err - } - err = Uncompress(src, g.folder) - if err != nil { - return err - } - // Gitlab zip/tar has contents inside a folder - // It has the format of eg. basic-micro-service-master-314b4a494ed472793e0a8bce8babbc69359aed7b - // Since we don't have the commit at this point we must list the dir - files, err := ioutil.ReadDir(g.folder) - if err != nil { - return err - } - if len(files) == 0 { - return fmt.Errorf("No contents in dir downloaded from gitlab: %v", g.folder) - } - g.folder = filepath.Join(g.folder, files[0].Name()) return nil } return fmt.Errorf("Repo host %v is not supported yet", repo) } +func (g *binaryGitter) checkoutGithub(repo, branchOrCommit string) error { + // @todo if it's a commit it must not be checked out all the time + repoFolder := strings.ReplaceAll(strings.ReplaceAll(repo, "/", "-"), "https://", "") + g.folder = filepath.Join(os.TempDir(), + repoFolder+"-"+shortid.MustGenerate()) + + url := fmt.Sprintf("%v/archive/%v.zip", repo, branchOrCommit) + if !strings.HasPrefix(url, "https://") { + url = "https://" + url + } + client := &http.Client{} + req, _ := http.NewRequest("GET", url, nil) + if len(g.secrets["GIT_CREDENTIALS"]) > 0 { + req.Header.Set("Authorization", "token "+g.secrets["GIT_CREDENTIALS"]) + } + resp, err := client.Do(req) + if err != nil { + return fmt.Errorf("Can't get zip: %v", err) + } + + defer resp.Body.Close() + // Github returns 404 for tar.gz files... + // but still gives back a proper file so ignoring status code + // for now. + //if resp.StatusCode != 200 { + // return errors.New("Status code was not 200") + //} + + src := g.folder + ".zip" + // Create the file + out, err := os.Create(src) + if err != nil { + return fmt.Errorf("Can't create source file %v src: %v", src, err) + } + defer out.Close() + + // Write the body to file + _, err = io.Copy(out, resp.Body) + if err != nil { + return err + } + return unzip(src, g.folder, true) +} + +func (g *binaryGitter) checkoutGitLabPublic(repo, branchOrCommit string) error { + // Example: https://gitlab.com/micro-test/basic-micro-service/-/archive/master/basic-micro-service-master.tar.gz + // @todo if it's a commit it must not be checked out all the time + repoFolder := strings.ReplaceAll(strings.ReplaceAll(repo, "/", "-"), "https://", "") + g.folder = filepath.Join(os.TempDir(), + repoFolder+"-"+shortid.MustGenerate()) + + tarName := strings.ReplaceAll(strings.ReplaceAll(repo, "gitlab.com/", ""), "/", "-") + url := fmt.Sprintf("%v/-/archive/%v/%v.tar.gz", repo, branchOrCommit, tarName) + if !strings.HasPrefix(url, "https://") { + url = "https://" + url + } + client := &http.Client{} + req, _ := http.NewRequest("GET", url, nil) + resp, err := client.Do(req) + if err != nil { + return fmt.Errorf("Can't get zip: %v", err) + } + + defer resp.Body.Close() + + src := g.folder + ".tar.gz" + // Create the file + out, err := os.Create(src) + if err != nil { + return fmt.Errorf("Can't create source file %v src: %v", src, err) + } + defer out.Close() + + // Write the body to file + _, err = io.Copy(out, resp.Body) + if err != nil { + return err + } + err = Uncompress(src, g.folder) + if err != nil { + return err + } + // Gitlab zip/tar has contents inside a folder + // It has the format of eg. basic-micro-service-master-314b4a494ed472793e0a8bce8babbc69359aed7b + // Since we don't have the commit at this point we must list the dir + files, err := ioutil.ReadDir(g.folder) + if err != nil { + return err + } + if len(files) == 0 { + return fmt.Errorf("No contents in dir downloaded from gitlab: %v", g.folder) + } + g.folder = filepath.Join(g.folder, files[0].Name()) + return nil +} + +func (g *binaryGitter) checkoutGitLabPrivate(repo, branchOrCommit string) error { + git, err := gitlab.NewClient(g.secrets["GIT_CREDENTIALS"]) + if err != nil { + return err + } + owned := true + projects, _, err := git.Projects.ListProjects(&gitlab.ListProjectsOptions{ + Owned: &owned, + }) + if err != nil { + return err + } + projectID := "" + for _, project := range projects { + if strings.Contains(repo, project.Name) { + projectID = fmt.Sprintf("%v", project.ID) + } + } + if len(projectID) == 0 { + return fmt.Errorf("Project id not found for repo %v", repo) + } + // Example URL: + // https://gitlab.com/api/v3/projects/0000000/repository/archive?private_token=XXXXXXXXXXXXXXXXXXXX + url := fmt.Sprintf("https://gitlab.com/api/v4/projects/%v/repository/archive?private_token=%v", projectID, g.secrets["GIT_CREDENTIALS"]) + + client := &http.Client{} + req, _ := http.NewRequest("GET", url, nil) + resp, err := client.Do(req) + if err != nil { + return fmt.Errorf("Can't get zip: %v", err) + } + + defer resp.Body.Close() + + src := g.folder + ".tar.gz" + // Create the file + out, err := os.Create(src) + if err != nil { + return fmt.Errorf("Can't create source file %v src: %v", src, err) + } + defer out.Close() + + // Write the body to file + _, err = io.Copy(out, resp.Body) + if err != nil { + return err + } + err = Uncompress(src, g.folder) + if err != nil { + return err + } + // Gitlab zip/tar has contents inside a folder + // It has the format of eg. basic-micro-service-master-314b4a494ed472793e0a8bce8babbc69359aed7b + // Since we don't have the commit at this point we must list the dir + files, err := ioutil.ReadDir(g.folder) + if err != nil { + return err + } + if len(files) == 0 { + return fmt.Errorf("No contents in dir downloaded from gitlab: %v", g.folder) + } + g.folder = filepath.Join(g.folder, files[0].Name()) + return nil +} + func (g *binaryGitter) RepoDir() string { return g.folder } -func NewGitter(folder string) Gitter { - return &binaryGitter{folder} +func NewGitter(folder string, secrets map[string]string) Gitter { + return &binaryGitter{folder, secrets} } @@ -299,12 +388,12 @@ func IsLocal(workDir, source string, pathExistsFunc ...func(path string) (bool, // CheckoutSource for the local runtime server // folder is the folder to check out the source code to // Modifies source path to set it to checked out repo absolute path locally. -func CheckoutSource(folder string, source *Source) error { +func CheckoutSource(folder string, source *Source, secrets map[string]string) error { // if it's a local folder, do nothing if exists, err := pathExists(source.FullPath); err == nil && exists { return nil } - gitter := NewGitter(folder) + gitter := NewGitter(folder, secrets) repo := source.Repo if !strings.Contains(repo, "https://") { repo = "https://" + repo diff --git a/runtime/local/local.go b/runtime/local/local.go index 9366846a..8ed52210 100644 --- a/runtime/local/local.go +++ b/runtime/local/local.go @@ -63,7 +63,7 @@ func NewRuntime(opts ...runtime.Option) runtime.Runtime { } } -func (r *localRuntime) checkoutSourceIfNeeded(s *runtime.Service) error { +func (r *localRuntime) checkoutSourceIfNeeded(s *runtime.Service, secrets map[string]string) error { // Runtime service like config have no source. // Skip checkout in that case if len(s.Source) == 0 { @@ -109,7 +109,7 @@ func (r *localRuntime) checkoutSourceIfNeeded(s *runtime.Service) error { } source.Ref = s.Version - err = git.CheckoutSource(os.TempDir(), source) + err = git.CheckoutSource(os.TempDir(), source, secrets) if err != nil { return err } @@ -271,17 +271,17 @@ func serviceKey(s *runtime.Service) string { // Create creates a new service which is then started by runtime func (r *localRuntime) Create(s *runtime.Service, opts ...runtime.CreateOption) error { - err := r.checkoutSourceIfNeeded(s) + var options runtime.CreateOptions + for _, o := range opts { + o(&options) + } + err := r.checkoutSourceIfNeeded(s, options.Secrets) if err != nil { return err } r.Lock() defer r.Unlock() - var options runtime.CreateOptions - for _, o := range opts { - o(&options) - } if len(options.Namespace) == 0 { options.Namespace = defaultNamespace } @@ -487,15 +487,15 @@ func (r *localRuntime) Update(s *runtime.Service, opts ...runtime.UpdateOption) for _, o := range opts { o(&options) } - if len(options.Namespace) == 0 { - options.Namespace = defaultNamespace - } - - err := r.checkoutSourceIfNeeded(s) + err := r.checkoutSourceIfNeeded(s, options.Secrets) if err != nil { return err } + if len(options.Namespace) == 0 { + options.Namespace = defaultNamespace + } + r.Lock() srvs, ok := r.namespaces[options.Namespace] r.Unlock() diff --git a/runtime/local/source/git/git.go b/runtime/local/source/git/git.go deleted file mode 100644 index 34b5a189..00000000 --- a/runtime/local/source/git/git.go +++ /dev/null @@ -1,87 +0,0 @@ -// Package git provides a git source -package git - -import ( - "os" - "path/filepath" - "strings" - - "github.com/go-git/go-git/v5" - "github.com/micro/go-micro/v3/runtime/local/source" -) - -// Source retrieves source code -// An empty struct can be used -type Source struct { - Options source.Options -} - -func (g *Source) Fetch(url string) (*source.Repository, error) { - purl := url - - if parts := strings.Split(url, "://"); len(parts) > 1 { - purl = parts[len(parts)-1] - } - - name := filepath.Base(url) - path := filepath.Join(g.Options.Path, purl) - - _, err := git.PlainClone(path, false, &git.CloneOptions{ - URL: url, - }) - if err == nil { - return &source.Repository{ - Name: name, - Path: path, - URL: url, - }, nil - } - - // repo already exists - if err != git.ErrRepositoryAlreadyExists { - return nil, err - } - - // open repo - re, err := git.PlainOpen(path) - if err != nil { - return nil, err - } - - // update it - if err := re.Fetch(nil); err != nil { - return nil, err - } - - return &source.Repository{ - Name: name, - Path: path, - URL: url, - }, nil -} - -func (g *Source) Commit(r *source.Repository) error { - repo := filepath.Join(r.Path) - re, err := git.PlainOpen(repo) - if err != nil { - return err - } - return re.Push(nil) -} - -func (g *Source) String() string { - return "git" -} - -func NewSource(opts ...source.Option) *Source { - options := source.Options{ - Path: os.TempDir(), - } - for _, o := range opts { - o(&options) - } - - return &Source{ - Options: options, - } -} diff --git a/runtime/options.go b/runtime/options.go index dd5cc23d..e3097acb 100644 --- a/runtime/options.go +++ b/runtime/options.go @@ -227,6 +227,19 @@ type UpdateOptions struct { Namespace string // Specify the context to use Context context.Context + // Secrets to use + Secrets map[string]string +} + +// WithSecret sets a secret to provide the service with +func UpdateSecret(key, value string) UpdateOption { + return func(o *UpdateOptions) { + if o.Secrets == nil { + o.Secrets = map[string]string{key: value} + } else { + o.Secrets[key] = value + } + } } // UpdateNamespace sets the namespace