unistack-org/micro
3
0
Fork 3
Code Issues 16 Pull Requests 2 Actions Packages Projects Releases 239 Wiki Activity

Support private repos in env 'local' (#1938)

Browse Source
This commit is contained in:
Janos Dobronszki 2020-08-18 18:26:14 +02:00 committed by GitHub
parent 21cca297c0
commit 2b2dc2f811
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 207 additions and 185 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
go.mod
View File

@ -56,12 +56,12 @@ require (
github.com/stretchr/testify v1.5.1
github.com/teris-io/shortid v0.0.0-20171029131806-771a37caa5cf
github.com/tmc/grpc-websocket-proxy v0.0.0-20200122045848-3419fae592fc // indirect
github.com/xanzy/go-gitlab v0.35.1 // indirect
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect
go.etcd.io/bbolt v1.3.5
go.uber.org/zap v1.13.0
golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899
golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2
golang.org/x/time v0.0.0-20191024005414-555d28b269f0 // indirect
golang.org/x/tools v0.0.0-20200117065230-39095c1d176c // indirect
google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013
google.golang.org/grpc v1.27.0

7
go.sum
View File

@ -214,6 +214,7 @@ github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-cmp v0.5.0 h1:/QaMHBdZ26BB3SSst0Iwl10Epc+xhTquomWX0oZEB6w=
github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk=
github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
@ -471,6 +472,8 @@ github.com/uber-go/atomic v1.3.2/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex
github.com/urfave/cli v0.0.0-20171014202726-7bc6a0acffa5/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
github.com/vultr/govultr v0.1.4/go.mod h1:9H008Uxr/C4vFNGLqKx232C206GL0PBHzOP0809bGNA=
github.com/xanzy/go-gitlab v0.35.1 h1:jJSgT0NxjCvrSZf7Gvn2NxxV9xAYkTjYrKW8XwWhrfY=
github.com/xanzy/go-gitlab v0.35.1/go.mod h1:sPLojNBn68fMUWSxIJtdVVIP8uSBYqesTfDUseX11Ug=
github.com/xanzy/ssh-agent v0.2.1 h1:TCbipTQL2JiiCprBWx9frJ2eJlCYT00NmctrHxVAr70=
github.com/xanzy/ssh-agent v0.2.1/go.mod h1:mLlQY/MoOhWBj+gOGMQkOeiEvkx+8pJSI+0Bx9h2kr4=
github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
@ -538,6 +541,7 @@ golang.org/x/net v0.0.0-20180611182652-db08ff08e862/go.mod h1:mL1N/T3taQHkDXs73r
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181108082009-03003ca0c849/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@ -560,7 +564,9 @@ golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLL
golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2 h1:eDrdRpKgkcCqKZQwyZRyeFZgfqt37SL7Kv3tok06cKE=
golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20181106182150-f42d05182288/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0=
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@ -645,6 +651,7 @@ google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEn
google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=

259
runtime/local/git/git.go
View File

@ -16,6 +16,7 @@ import (
"strings"
"github.com/teris-io/shortid"
"github.com/xanzy/go-gitlab"
)
type Gitter interface {
@ -24,7 +25,8 @@ type Gitter interface {
}
type binaryGitter struct {
folder string
folder string
secrets map[string]string
}
func (g *binaryGitter) Checkout(repo, branchOrCommit string) error {
@ -38,100 +40,187 @@ func (g *binaryGitter) Checkout(repo, branchOrCommit string) error {
branchOrCommit = "master"
}
if strings.Contains(repo, "github") {
// @todo if it's a commit it must not be checked out all the time
repoFolder := strings.ReplaceAll(strings.ReplaceAll(repo, "/", "-"), "https://", "")
g.folder = filepath.Join(os.TempDir(),
repoFolder+"-"+shortid.MustGenerate())
url := fmt.Sprintf("%v/archive/%v.zip", repo, branchOrCommit)
if !strings.HasPrefix(url, "https://") {
url = "https://" + url
}
resp, err := http.Get(url)
if err != nil {
return fmt.Errorf("Can't get zip: %v", err)
}
defer resp.Body.Close()
// Github returns 404 for tar.gz files...
// but still gives back a proper file so ignoring status code
// for now.
//if resp.StatusCode != 200 {
// return errors.New("Status code was not 200")
//}
src := g.folder + ".zip"
// Create the file
out, err := os.Create(src)
if err != nil {
return fmt.Errorf("Can't create source file %v src: %v", src, err)
}
defer out.Close()
// Write the body to file
_, err = io.Copy(out, resp.Body)
if err != nil {
return err
}
return unzip(src, g.folder, true)
return g.checkoutGithub(repo, branchOrCommit)
} else if strings.Contains(repo, "gitlab") {
// Example: https://gitlab.com/micro-test/basic-micro-service/-/archive/master/basic-micro-service-master.tar.gz
// @todo if it's a commit it must not be checked out all the time
repoFolder := strings.ReplaceAll(strings.ReplaceAll(repo, "/", "-"), "https://", "")
g.folder = filepath.Join(os.TempDir(),
repoFolder+"-"+shortid.MustGenerate())
tarName := strings.ReplaceAll(strings.ReplaceAll(repo, "gitlab.com/", ""), "/", "-")
url := fmt.Sprintf("%v/-/archive/%v/%v.tar.gz", repo, branchOrCommit, tarName)
if !strings.HasPrefix(url, "https://") {
url = "https://" + url
}
resp, err := http.Get(url)
err := g.checkoutGitLabPublic(repo, branchOrCommit)
if err != nil {
return fmt.Errorf("Can't get zip: %v", err)
// If the public download fails, try getting it with tokens.
// Private downloads needs a token for api project listing, hence
// the weird structure of this code.
return g.checkoutGitLabPrivate(repo, branchOrCommit)
}
defer resp.Body.Close()
src := g.folder + ".tar.gz"
// Create the file
out, err := os.Create(src)
if err != nil {
return fmt.Errorf("Can't create source file %v src: %v", src, err)
}
defer out.Close()
// Write the body to file
_, err = io.Copy(out, resp.Body)
if err != nil {
return err
}
err = Uncompress(src, g.folder)
if err != nil {
return err
}
// Gitlab zip/tar has contents inside a folder
// It has the format of eg. basic-micro-service-master-314b4a494ed472793e0a8bce8babbc69359aed7b
// Since we don't have the commit at this point we must list the dir
files, err := ioutil.ReadDir(g.folder)
if err != nil {
return err
}
if len(files) == 0 {
return fmt.Errorf("No contents in dir downloaded from gitlab: %v", g.folder)
}
g.folder = filepath.Join(g.folder, files[0].Name())
return nil
}
return fmt.Errorf("Repo host %v is not supported yet", repo)
}
func (g *binaryGitter) checkoutGithub(repo, branchOrCommit string) error {
// @todo if it's a commit it must not be checked out all the time
repoFolder := strings.ReplaceAll(strings.ReplaceAll(repo, "/", "-"), "https://", "")
g.folder = filepath.Join(os.TempDir(),
repoFolder+"-"+shortid.MustGenerate())
url := fmt.Sprintf("%v/archive/%v.zip", repo, branchOrCommit)
if !strings.HasPrefix(url, "https://") {
url = "https://" + url
}
client := &http.Client{}
req, _ := http.NewRequest("GET", url, nil)
if len(g.secrets["GIT_CREDENTIALS"]) > 0 {
req.Header.Set("Authorization", "token "+g.secrets["GIT_CREDENTIALS"])
}
resp, err := client.Do(req)
if err != nil {
return fmt.Errorf("Can't get zip: %v", err)
}
defer resp.Body.Close()
// Github returns 404 for tar.gz files...
// but still gives back a proper file so ignoring status code
// for now.
//if resp.StatusCode != 200 {
// return errors.New("Status code was not 200")
//}
src := g.folder + ".zip"
// Create the file
out, err := os.Create(src)
if err != nil {
return fmt.Errorf("Can't create source file %v src: %v", src, err)
}
defer out.Close()
// Write the body to file
_, err = io.Copy(out, resp.Body)
if err != nil {
return err
}
return unzip(src, g.folder, true)
}
func (g *binaryGitter) checkoutGitLabPublic(repo, branchOrCommit string) error {
// Example: https://gitlab.com/micro-test/basic-micro-service/-/archive/master/basic-micro-service-master.tar.gz
// @todo if it's a commit it must not be checked out all the time
repoFolder := strings.ReplaceAll(strings.ReplaceAll(repo, "/", "-"), "https://", "")
g.folder = filepath.Join(os.TempDir(),
repoFolder+"-"+shortid.MustGenerate())
tarName := strings.ReplaceAll(strings.ReplaceAll(repo, "gitlab.com/", ""), "/", "-")
url := fmt.Sprintf("%v/-/archive/%v/%v.tar.gz", repo, branchOrCommit, tarName)
if !strings.HasPrefix(url, "https://") {
url = "https://" + url
}
client := &http.Client{}
req, _ := http.NewRequest("GET", url, nil)
resp, err := client.Do(req)
if err != nil {
return fmt.Errorf("Can't get zip: %v", err)
}
defer resp.Body.Close()
src := g.folder + ".tar.gz"
// Create the file
out, err := os.Create(src)
if err != nil {
return fmt.Errorf("Can't create source file %v src: %v", src, err)
}
defer out.Close()
// Write the body to file
_, err = io.Copy(out, resp.Body)
if err != nil {
return err
}
err = Uncompress(src, g.folder)
if err != nil {
return err
}
// Gitlab zip/tar has contents inside a folder
// It has the format of eg. basic-micro-service-master-314b4a494ed472793e0a8bce8babbc69359aed7b
// Since we don't have the commit at this point we must list the dir
files, err := ioutil.ReadDir(g.folder)
if err != nil {
return err
}
if len(files) == 0 {
return fmt.Errorf("No contents in dir downloaded from gitlab: %v", g.folder)
}
g.folder = filepath.Join(g.folder, files[0].Name())
return nil
}
func (g *binaryGitter) checkoutGitLabPrivate(repo, branchOrCommit string) error {
git, err := gitlab.NewClient(g.secrets["GIT_CREDENTIALS"])
if err != nil {
return err
}
owned := true
projects, _, err := git.Projects.ListProjects(&gitlab.ListProjectsOptions{
Owned: &owned,
})
if err != nil {
return err
}
projectID := ""
for _, project := range projects {
if strings.Contains(repo, project.Name) {
projectID = fmt.Sprintf("%v", project.ID)
}
}
if len(projectID) == 0 {
return fmt.Errorf("Project id not found for repo %v", repo)
}
// Example URL:
// https://gitlab.com/api/v3/projects/0000000/repository/archive?private_token=XXXXXXXXXXXXXXXXXXXX
url := fmt.Sprintf("https://gitlab.com/api/v4/projects/%v/repository/archive?private_token=%v", projectID, g.secrets["GIT_CREDENTIALS"])
client := &http.Client{}
req, _ := http.NewRequest("GET", url, nil)
resp, err := client.Do(req)
if err != nil {
return fmt.Errorf("Can't get zip: %v", err)
}
defer resp.Body.Close()
src := g.folder + ".tar.gz"
// Create the file
out, err := os.Create(src)
if err != nil {
return fmt.Errorf("Can't create source file %v src: %v", src, err)
}
defer out.Close()
// Write the body to file
_, err = io.Copy(out, resp.Body)
if err != nil {
return err
}
err = Uncompress(src, g.folder)
if err != nil {
return err
}
// Gitlab zip/tar has contents inside a folder
// It has the format of eg. basic-micro-service-master-314b4a494ed472793e0a8bce8babbc69359aed7b
// Since we don't have the commit at this point we must list the dir
files, err := ioutil.ReadDir(g.folder)
if err != nil {
return err
}
if len(files) == 0 {
return fmt.Errorf("No contents in dir downloaded from gitlab: %v", g.folder)
}
g.folder = filepath.Join(g.folder, files[0].Name())
return nil
}
func (g *binaryGitter) RepoDir() string {
return g.folder
}
func NewGitter(folder string) Gitter {
return &binaryGitter{folder}
func NewGitter(folder string, secrets map[string]string) Gitter {
return &binaryGitter{folder, secrets}
}
@ -299,12 +388,12 @@ func IsLocal(workDir, source string, pathExistsFunc ...func(path string) (bool,
// CheckoutSource for the local runtime server
// folder is the folder to check out the source code to
// Modifies source path to set it to checked out repo absolute path locally.
func CheckoutSource(folder string, source *Source) error {
func CheckoutSource(folder string, source *Source, secrets map[string]string) error {
// if it's a local folder, do nothing
if exists, err := pathExists(source.FullPath); err == nil && exists {
return nil
}
gitter := NewGitter(folder)
gitter := NewGitter(folder, secrets)
repo := source.Repo
if !strings.Contains(repo, "https://") {
repo = "https://" + repo

24
runtime/local/local.go
View File

@ -63,7 +63,7 @@ func NewRuntime(opts ...runtime.Option) runtime.Runtime {
}
}
func (r *localRuntime) checkoutSourceIfNeeded(s *runtime.Service) error {
func (r *localRuntime) checkoutSourceIfNeeded(s *runtime.Service, secrets map[string]string) error {
// Runtime service like config have no source.
// Skip checkout in that case
if len(s.Source) == 0 {
@ -109,7 +109,7 @@ func (r *localRuntime) checkoutSourceIfNeeded(s *runtime.Service) error {
}
source.Ref = s.Version
err = git.CheckoutSource(os.TempDir(), source)
err = git.CheckoutSource(os.TempDir(), source, secrets)
if err != nil {
return err
}
@ -271,17 +271,17 @@ func serviceKey(s *runtime.Service) string {
// Create creates a new service which is then started by runtime
func (r *localRuntime) Create(s *runtime.Service, opts ...runtime.CreateOption) error {
err := r.checkoutSourceIfNeeded(s)
var options runtime.CreateOptions
for _, o := range opts {
o(&options)
}
err := r.checkoutSourceIfNeeded(s, options.Secrets)
if err != nil {
return err
}
r.Lock()
defer r.Unlock()
var options runtime.CreateOptions
for _, o := range opts {
o(&options)
}
if len(options.Namespace) == 0 {
options.Namespace = defaultNamespace
}
@ -487,15 +487,15 @@ func (r *localRuntime) Update(s *runtime.Service, opts ...runtime.UpdateOption)
for _, o := range opts {
o(&options)
}
if len(options.Namespace) == 0 {
options.Namespace = defaultNamespace
}
err := r.checkoutSourceIfNeeded(s)
err := r.checkoutSourceIfNeeded(s, options.Secrets)
if err != nil {
return err
}
if len(options.Namespace) == 0 {
options.Namespace = defaultNamespace
}
r.Lock()
srvs, ok := r.namespaces[options.Namespace]
r.Unlock()

87
runtime/local/source/git/git.go
View File

@ -1,87 +0,0 @@
// Package git provides a git source
package git
import (
"os"
"path/filepath"
"strings"
"github.com/go-git/go-git/v5"
"github.com/micro/go-micro/v3/runtime/local/source"
)
// Source retrieves source code
// An empty struct can be used
type Source struct {
Options source.Options
}
func (g *Source) Fetch(url string) (*source.Repository, error) {
purl := url
if parts := strings.Split(url, "://"); len(parts) > 1 {
purl = parts[len(parts)-1]
}
name := filepath.Base(url)
path := filepath.Join(g.Options.Path, purl)
_, err := git.PlainClone(path, false, &git.CloneOptions{
URL: url,
})
if err == nil {
return &source.Repository{
Name: name,
Path: path,
URL: url,
}, nil
}
// repo already exists
if err != git.ErrRepositoryAlreadyExists {
return nil, err
}
// open repo
re, err := git.PlainOpen(path)
if err != nil {
return nil, err
}
// update it
if err := re.Fetch(nil); err != nil {
return nil, err
}
return &source.Repository{
Name: name,
Path: path,
URL: url,
}, nil
}
func (g *Source) Commit(r *source.Repository) error {
repo := filepath.Join(r.Path)
re, err := git.PlainOpen(repo)
if err != nil {
return err
}
return re.Push(nil)
}
func (g *Source) String() string {
return "git"
}
func NewSource(opts ...source.Option) *Source {
options := source.Options{
Path: os.TempDir(),
}
for _, o := range opts {
o(&options)
}
return &Source{
Options: options,
}
}

13
runtime/options.go
View File

@ -227,6 +227,19 @@ type UpdateOptions struct {
Namespace string
// Specify the context to use
Context context.Context
// Secrets to use
Secrets map[string]string
}
// WithSecret sets a secret to provide the service with
func UpdateSecret(key, value string) UpdateOption {
return func(o *UpdateOptions) {
if o.Secrets == nil {
o.Secrets = map[string]string{key: value}
} else {
o.Secrets[key] = value
}
}
}
// UpdateNamespace sets the namespace