Support private repos in env 'local' (#1938)
This commit is contained in:
Janos Dobronszki
GitHub
parent
21cca297c0
commit
2b2dc2f811
2
go.mod
2
go.mod
@ -56,12 +56,12 @@ require (
|
||||
github.com/stretchr/testify v1.5.1
|
||||
github.com/teris-io/shortid v0.0.0-20171029131806-771a37caa5cf
|
||||
github.com/tmc/grpc-websocket-proxy v0.0.0-20200122045848-3419fae592fc // indirect
|
||||
github.com/xanzy/go-gitlab v0.35.1 // indirect
|
||||
github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect
|
||||
go.etcd.io/bbolt v1.3.5
|
||||
go.uber.org/zap v1.13.0
|
||||
golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899
|
||||
golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2
|
||||
golang.org/x/time v0.0.0-20191024005414-555d28b269f0 // indirect
|
||||
golang.org/x/tools v0.0.0-20200117065230-39095c1d176c // indirect
|
||||
google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013
|
||||
google.golang.org/grpc v1.27.0
|
||||
|
||||
7
go.sum
7
go.sum
@ -214,6 +214,7 @@ github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
|
||||
github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-cmp v0.5.0 h1:/QaMHBdZ26BB3SSst0Iwl10Epc+xhTquomWX0oZEB6w=
|
||||
github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk=
|
||||
github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
|
||||
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
|
||||
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
|
||||
@ -471,6 +472,8 @@ github.com/uber-go/atomic v1.3.2/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex
|
||||
github.com/urfave/cli v0.0.0-20171014202726-7bc6a0acffa5/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
|
||||
github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
|
||||
github.com/vultr/govultr v0.1.4/go.mod h1:9H008Uxr/C4vFNGLqKx232C206GL0PBHzOP0809bGNA=
|
||||
github.com/xanzy/go-gitlab v0.35.1 h1:jJSgT0NxjCvrSZf7Gvn2NxxV9xAYkTjYrKW8XwWhrfY=
|
||||
github.com/xanzy/go-gitlab v0.35.1/go.mod h1:sPLojNBn68fMUWSxIJtdVVIP8uSBYqesTfDUseX11Ug=
|
||||
github.com/xanzy/ssh-agent v0.2.1 h1:TCbipTQL2JiiCprBWx9frJ2eJlCYT00NmctrHxVAr70=
|
||||
github.com/xanzy/ssh-agent v0.2.1/go.mod h1:mLlQY/MoOhWBj+gOGMQkOeiEvkx+8pJSI+0Bx9h2kr4=
|
||||
github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
|
||||
@ -538,6 +541,7 @@ golang.org/x/net v0.0.0-20180611182652-db08ff08e862/go.mod h1:mL1N/T3taQHkDXs73r
|
||||
golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20181108082009-03003ca0c849/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
|
||||
@ -560,7 +564,9 @@ golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLL
|
||||
golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2 h1:eDrdRpKgkcCqKZQwyZRyeFZgfqt37SL7Kv3tok06cKE=
|
||||
golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
|
||||
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/oauth2 v0.0.0-20181106182150-f42d05182288/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
|
||||
golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0=
|
||||
golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
|
||||
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
@ -645,6 +651,7 @@ google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEn
|
||||
google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
|
||||
google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
|
||||
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
|
||||
google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
|
||||
google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=
|
||||
|
||||
@ -16,6 +16,7 @@ import (
|
||||
"strings"
|
||||
|
||||
"github.com/teris-io/shortid"
|
||||
"github.com/xanzy/go-gitlab"
|
||||
)
|
||||
|
||||
type Gitter interface {
|
||||
@ -25,6 +26,7 @@ type Gitter interface {
|
||||
|
||||
type binaryGitter struct {
|
||||
folder string
|
||||
secrets map[string]string
|
||||
}
|
||||
|
||||
func (g *binaryGitter) Checkout(repo, branchOrCommit string) error {
|
||||
@ -38,6 +40,21 @@ func (g *binaryGitter) Checkout(repo, branchOrCommit string) error {
|
||||
branchOrCommit = "master"
|
||||
}
|
||||
if strings.Contains(repo, "github") {
|
||||
return g.checkoutGithub(repo, branchOrCommit)
|
||||
} else if strings.Contains(repo, "gitlab") {
|
||||
err := g.checkoutGitLabPublic(repo, branchOrCommit)
|
||||
if err != nil {
|
||||
// If the public download fails, try getting it with tokens.
|
||||
// Private downloads needs a token for api project listing, hence
|
||||
// the weird structure of this code.
|
||||
return g.checkoutGitLabPrivate(repo, branchOrCommit)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("Repo host %v is not supported yet", repo)
|
||||
}
|
||||
|
||||
func (g *binaryGitter) checkoutGithub(repo, branchOrCommit string) error {
|
||||
// @todo if it's a commit it must not be checked out all the time
|
||||
repoFolder := strings.ReplaceAll(strings.ReplaceAll(repo, "/", "-"), "https://", "")
|
||||
g.folder = filepath.Join(os.TempDir(),
|
||||
@ -47,7 +64,12 @@ func (g *binaryGitter) Checkout(repo, branchOrCommit string) error {
|
||||
if !strings.HasPrefix(url, "https://") {
|
||||
url = "https://" + url
|
||||
}
|
||||
resp, err := http.Get(url)
|
||||
client := &http.Client{}
|
||||
req, _ := http.NewRequest("GET", url, nil)
|
||||
if len(g.secrets["GIT_CREDENTIALS"]) > 0 {
|
||||
req.Header.Set("Authorization", "token "+g.secrets["GIT_CREDENTIALS"])
|
||||
}
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Can't get zip: %v", err)
|
||||
}
|
||||
@ -74,7 +96,9 @@ func (g *binaryGitter) Checkout(repo, branchOrCommit string) error {
|
||||
return err
|
||||
}
|
||||
return unzip(src, g.folder, true)
|
||||
} else if strings.Contains(repo, "gitlab") {
|
||||
}
|
||||
|
||||
func (g *binaryGitter) checkoutGitLabPublic(repo, branchOrCommit string) error {
|
||||
// Example: https://gitlab.com/micro-test/basic-micro-service/-/archive/master/basic-micro-service-master.tar.gz
|
||||
// @todo if it's a commit it must not be checked out all the time
|
||||
repoFolder := strings.ReplaceAll(strings.ReplaceAll(repo, "/", "-"), "https://", "")
|
||||
@ -86,7 +110,9 @@ func (g *binaryGitter) Checkout(repo, branchOrCommit string) error {
|
||||
if !strings.HasPrefix(url, "https://") {
|
||||
url = "https://" + url
|
||||
}
|
||||
resp, err := http.Get(url)
|
||||
client := &http.Client{}
|
||||
req, _ := http.NewRequest("GET", url, nil)
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Can't get zip: %v", err)
|
||||
}
|
||||
@ -122,16 +148,79 @@ func (g *binaryGitter) Checkout(repo, branchOrCommit string) error {
|
||||
}
|
||||
g.folder = filepath.Join(g.folder, files[0].Name())
|
||||
return nil
|
||||
}
|
||||
|
||||
func (g *binaryGitter) checkoutGitLabPrivate(repo, branchOrCommit string) error {
|
||||
git, err := gitlab.NewClient(g.secrets["GIT_CREDENTIALS"])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return fmt.Errorf("Repo host %v is not supported yet", repo)
|
||||
owned := true
|
||||
projects, _, err := git.Projects.ListProjects(&gitlab.ListProjectsOptions{
|
||||
Owned: &owned,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
projectID := ""
|
||||
for _, project := range projects {
|
||||
if strings.Contains(repo, project.Name) {
|
||||
projectID = fmt.Sprintf("%v", project.ID)
|
||||
}
|
||||
}
|
||||
if len(projectID) == 0 {
|
||||
return fmt.Errorf("Project id not found for repo %v", repo)
|
||||
}
|
||||
// Example URL:
|
||||
// https://gitlab.com/api/v3/projects/0000000/repository/archive?private_token=XXXXXXXXXXXXXXXXXXXX
|
||||
url := fmt.Sprintf("https://gitlab.com/api/v4/projects/%v/repository/archive?private_token=%v", projectID, g.secrets["GIT_CREDENTIALS"])
|
||||
|
||||
client := &http.Client{}
|
||||
req, _ := http.NewRequest("GET", url, nil)
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Can't get zip: %v", err)
|
||||
}
|
||||
|
||||
defer resp.Body.Close()
|
||||
|
||||
src := g.folder + ".tar.gz"
|
||||
// Create the file
|
||||
out, err := os.Create(src)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Can't create source file %v src: %v", src, err)
|
||||
}
|
||||
defer out.Close()
|
||||
|
||||
// Write the body to file
|
||||
_, err = io.Copy(out, resp.Body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = Uncompress(src, g.folder)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// Gitlab zip/tar has contents inside a folder
|
||||
// It has the format of eg. basic-micro-service-master-314b4a494ed472793e0a8bce8babbc69359aed7b
|
||||
// Since we don't have the commit at this point we must list the dir
|
||||
files, err := ioutil.ReadDir(g.folder)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(files) == 0 {
|
||||
return fmt.Errorf("No contents in dir downloaded from gitlab: %v", g.folder)
|
||||
}
|
||||
g.folder = filepath.Join(g.folder, files[0].Name())
|
||||
return nil
|
||||
}
|
||||
|
||||
func (g *binaryGitter) RepoDir() string {
|
||||
return g.folder
|
||||
}
|
||||
|
||||
func NewGitter(folder string) Gitter {
|
||||
return &binaryGitter{folder}
|
||||
func NewGitter(folder string, secrets map[string]string) Gitter {
|
||||
return &binaryGitter{folder, secrets}
|
||||
|
||||
}
|
||||
|
||||
@ -299,12 +388,12 @@ func IsLocal(workDir, source string, pathExistsFunc ...func(path string) (bool,
|
||||
// CheckoutSource for the local runtime server
|
||||
// folder is the folder to check out the source code to
|
||||
// Modifies source path to set it to checked out repo absolute path locally.
|
||||
func CheckoutSource(folder string, source *Source) error {
|
||||
func CheckoutSource(folder string, source *Source, secrets map[string]string) error {
|
||||
// if it's a local folder, do nothing
|
||||
if exists, err := pathExists(source.FullPath); err == nil && exists {
|
||||
return nil
|
||||
}
|
||||
gitter := NewGitter(folder)
|
||||
gitter := NewGitter(folder, secrets)
|
||||
repo := source.Repo
|
||||
if !strings.Contains(repo, "https://") {
|
||||
repo = "https://" + repo
|
||||
|
||||
@ -63,7 +63,7 @@ func NewRuntime(opts ...runtime.Option) runtime.Runtime {
|
||||
}
|
||||
}
|
||||
|
||||
func (r *localRuntime) checkoutSourceIfNeeded(s *runtime.Service) error {
|
||||
func (r *localRuntime) checkoutSourceIfNeeded(s *runtime.Service, secrets map[string]string) error {
|
||||
// Runtime service like config have no source.
|
||||
// Skip checkout in that case
|
||||
if len(s.Source) == 0 {
|
||||
@ -109,7 +109,7 @@ func (r *localRuntime) checkoutSourceIfNeeded(s *runtime.Service) error {
|
||||
}
|
||||
source.Ref = s.Version
|
||||
|
||||
err = git.CheckoutSource(os.TempDir(), source)
|
||||
err = git.CheckoutSource(os.TempDir(), source, secrets)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@ -271,17 +271,17 @@ func serviceKey(s *runtime.Service) string {
|
||||
|
||||
// Create creates a new service which is then started by runtime
|
||||
func (r *localRuntime) Create(s *runtime.Service, opts ...runtime.CreateOption) error {
|
||||
err := r.checkoutSourceIfNeeded(s)
|
||||
var options runtime.CreateOptions
|
||||
for _, o := range opts {
|
||||
o(&options)
|
||||
}
|
||||
err := r.checkoutSourceIfNeeded(s, options.Secrets)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
r.Lock()
|
||||
defer r.Unlock()
|
||||
|
||||
var options runtime.CreateOptions
|
||||
for _, o := range opts {
|
||||
o(&options)
|
||||
}
|
||||
if len(options.Namespace) == 0 {
|
||||
options.Namespace = defaultNamespace
|
||||
}
|
||||
@ -487,15 +487,15 @@ func (r *localRuntime) Update(s *runtime.Service, opts ...runtime.UpdateOption)
|
||||
for _, o := range opts {
|
||||
o(&options)
|
||||
}
|
||||
if len(options.Namespace) == 0 {
|
||||
options.Namespace = defaultNamespace
|
||||
}
|
||||
|
||||
err := r.checkoutSourceIfNeeded(s)
|
||||
err := r.checkoutSourceIfNeeded(s, options.Secrets)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if len(options.Namespace) == 0 {
|
||||
options.Namespace = defaultNamespace
|
||||
}
|
||||
|
||||
r.Lock()
|
||||
srvs, ok := r.namespaces[options.Namespace]
|
||||
r.Unlock()
|
||||
|
||||
@ -1,87 +0,0 @@
|
||||
// Package git provides a git source
|
||||
package git
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/go-git/go-git/v5"
|
||||
"github.com/micro/go-micro/v3/runtime/local/source"
|
||||
)
|
||||
|
||||
// Source retrieves source code
|
||||
// An empty struct can be used
|
||||
type Source struct {
|
||||
Options source.Options
|
||||
}
|
||||
|
||||
func (g *Source) Fetch(url string) (*source.Repository, error) {
|
||||
purl := url
|
||||
|
||||
if parts := strings.Split(url, "://"); len(parts) > 1 {
|
||||
purl = parts[len(parts)-1]
|
||||
}
|
||||
|
||||
name := filepath.Base(url)
|
||||
path := filepath.Join(g.Options.Path, purl)
|
||||
|
||||
_, err := git.PlainClone(path, false, &git.CloneOptions{
|
||||
URL: url,
|
||||
})
|
||||
if err == nil {
|
||||
return &source.Repository{
|
||||
Name: name,
|
||||
Path: path,
|
||||
URL: url,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// repo already exists
|
||||
if err != git.ErrRepositoryAlreadyExists {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// open repo
|
||||
re, err := git.PlainOpen(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// update it
|
||||
if err := re.Fetch(nil); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &source.Repository{
|
||||
Name: name,
|
||||
Path: path,
|
||||
URL: url,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (g *Source) Commit(r *source.Repository) error {
|
||||
repo := filepath.Join(r.Path)
|
||||
re, err := git.PlainOpen(repo)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return re.Push(nil)
|
||||
}
|
||||
|
||||
func (g *Source) String() string {
|
||||
return "git"
|
||||
}
|
||||
|
||||
func NewSource(opts ...source.Option) *Source {
|
||||
options := source.Options{
|
||||
Path: os.TempDir(),
|
||||
}
|
||||
for _, o := range opts {
|
||||
o(&options)
|
||||
}
|
||||
|
||||
return &Source{
|
||||
Options: options,
|
||||
}
|
||||
}
|
||||
@ -227,6 +227,19 @@ type UpdateOptions struct {
|
||||
Namespace string
|
||||
// Specify the context to use
|
||||
Context context.Context
|
||||
// Secrets to use
|
||||
Secrets map[string]string
|
||||
}
|
||||
|
||||
// WithSecret sets a secret to provide the service with
|
||||
func UpdateSecret(key, value string) UpdateOption {
|
||||
return func(o *UpdateOptions) {
|
||||
if o.Secrets == nil {
|
||||
o.Secrets = map[string]string{key: value}
|
||||
} else {
|
||||
o.Secrets[key] = value
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// UpdateNamespace sets the namespace
|
||||
|
||||
Loading…
Reference in New Issue
Block a user