Support private repos in env 'local' (#1938)

go.mod

@@ -56,12 +56,12 @@ require (
 	github.com/stretchr/testify v1.5.1
 	github.com/teris-io/shortid v0.0.0-20171029131806-771a37caa5cf
 	github.com/tmc/grpc-websocket-proxy v0.0.0-20200122045848-3419fae592fc // indirect
+	github.com/xanzy/go-gitlab v0.35.1 // indirect
 	github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect
 	go.etcd.io/bbolt v1.3.5
 	go.uber.org/zap v1.13.0
 	golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899
 	golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2
-	golang.org/x/time v0.0.0-20191024005414-555d28b269f0 // indirect
 	golang.org/x/tools v0.0.0-20200117065230-39095c1d176c // indirect
 	google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013
 	google.golang.org/grpc v1.27.0

go.sum

@@ -214,6 +214,7 @@ github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4=
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.0 h1:/QaMHBdZ26BB3SSst0Iwl10Epc+xhTquomWX0oZEB6w=
 github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-querystring v1.0.0 h1:Xkwi/a1rcvNg1PPYe5vI8GbeBY/jrVuDX5ASuANWTrk=
 github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
@@ -471,6 +472,8 @@ github.com/uber-go/atomic v1.3.2/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex
 github.com/urfave/cli v0.0.0-20171014202726-7bc6a0acffa5/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA=
 github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0=
 github.com/vultr/govultr v0.1.4/go.mod h1:9H008Uxr/C4vFNGLqKx232C206GL0PBHzOP0809bGNA=
+github.com/xanzy/go-gitlab v0.35.1 h1:jJSgT0NxjCvrSZf7Gvn2NxxV9xAYkTjYrKW8XwWhrfY=
+github.com/xanzy/go-gitlab v0.35.1/go.mod h1:sPLojNBn68fMUWSxIJtdVVIP8uSBYqesTfDUseX11Ug=
 github.com/xanzy/ssh-agent v0.2.1 h1:TCbipTQL2JiiCprBWx9frJ2eJlCYT00NmctrHxVAr70=
 github.com/xanzy/ssh-agent v0.2.1/go.mod h1:mLlQY/MoOhWBj+gOGMQkOeiEvkx+8pJSI+0Bx9h2kr4=
 github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
@@ -538,6 +541,7 @@ golang.org/x/net v0.0.0-20180611182652-db08ff08e862/go.mod h1:mL1N/T3taQHkDXs73r
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20181108082009-03003ca0c849/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -560,7 +564,9 @@ golang.org/x/net v0.0.0-20200301022130-244492dfa37a/go.mod h1:z5CRVTTTmAJ677TzLL
 golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2 h1:eDrdRpKgkcCqKZQwyZRyeFZgfqt37SL7Kv3tok06cKE=
 golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/oauth2 v0.0.0-20181106182150-f42d05182288/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
+golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45 h1:SVwTIAaPC2U/AvvLNZ2a7OVsmBpC8L5BlwK1whH3hm0=
 golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -645,6 +651,7 @@ google.golang.org/api v0.8.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEn
 google.golang.org/api v0.9.0/go.mod h1:o4eAsZoiT+ibD93RtjEohWalFOjRDx6CVaqeizhEnKg=
 google.golang.org/api v0.14.0/go.mod h1:iLdEw5Ide6rF15KTC1Kkl0iskquN2gFfn9o9XIsbkAI=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
+google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0=

runtime/local/git/git.go

@@ -16,6 +16,7 @@ import (
 	"strings"
 
 	"github.com/teris-io/shortid"
+	"github.com/xanzy/go-gitlab"
 )
 
 type Gitter interface {
@@ -24,7 +25,8 @@ type Gitter interface {
 }
 
 type binaryGitter struct {
-	folder string
+	folder  string
+	secrets map[string]string
 }
 
 func (g *binaryGitter) Checkout(repo, branchOrCommit string) error {
@@ -38,100 +40,187 @@ func (g *binaryGitter) Checkout(repo, branchOrCommit string) error {
 		branchOrCommit = "master"
 	}
 	if strings.Contains(repo, "github") {
-		// @todo if it's a commit it must not be checked out all the time
+		return g.checkoutGithub(repo, branchOrCommit)
-		repoFolder := strings.ReplaceAll(strings.ReplaceAll(repo, "/", "-"), "https://", "")
-		g.folder = filepath.Join(os.TempDir(),
-			repoFolder+"-"+shortid.MustGenerate())
-
-		url := fmt.Sprintf("%v/archive/%v.zip", repo, branchOrCommit)
-		if !strings.HasPrefix(url, "https://") {
-			url = "https://" + url
-		}
-		resp, err := http.Get(url)
-		if err != nil {
-			return fmt.Errorf("Can't get zip: %v", err)
-		}
-
-		defer resp.Body.Close()
-		// Github returns 404 for tar.gz files...
-		// but still gives back a proper file so ignoring status code
-		// for now.
-		//if resp.StatusCode != 200 {
-		//	return errors.New("Status code was not 200")
-		//}
-
-		src := g.folder + ".zip"
-		// Create the file
-		out, err := os.Create(src)
-		if err != nil {
-			return fmt.Errorf("Can't create source file %v src: %v", src, err)
-		}
-		defer out.Close()
-
-		// Write the body to file
-		_, err = io.Copy(out, resp.Body)
-		if err != nil {
-			return err
-		}
-		return unzip(src, g.folder, true)
 	} else if strings.Contains(repo, "gitlab") {
-		// Example: https://gitlab.com/micro-test/basic-micro-service/-/archive/master/basic-micro-service-master.tar.gz
+		err := g.checkoutGitLabPublic(repo, branchOrCommit)
-		// @todo if it's a commit it must not be checked out all the time
-		repoFolder := strings.ReplaceAll(strings.ReplaceAll(repo, "/", "-"), "https://", "")
-		g.folder = filepath.Join(os.TempDir(),
-			repoFolder+"-"+shortid.MustGenerate())
-
-		tarName := strings.ReplaceAll(strings.ReplaceAll(repo, "gitlab.com/", ""), "/", "-")
-		url := fmt.Sprintf("%v/-/archive/%v/%v.tar.gz", repo, branchOrCommit, tarName)
-		if !strings.HasPrefix(url, "https://") {
-			url = "https://" + url
-		}
-		resp, err := http.Get(url)
 		if err != nil {
-			return fmt.Errorf("Can't get zip: %v", err)
+			// If the public download fails, try getting it with tokens.
+			// Private downloads needs a token for api project listing, hence
+			// the weird structure of this code.
+			return g.checkoutGitLabPrivate(repo, branchOrCommit)
 		}
-
-		defer resp.Body.Close()
-
-		src := g.folder + ".tar.gz"
-		// Create the file
-		out, err := os.Create(src)
-		if err != nil {
-			return fmt.Errorf("Can't create source file %v src: %v", src, err)
-		}
-		defer out.Close()
-
-		// Write the body to file
-		_, err = io.Copy(out, resp.Body)
-		if err != nil {
-			return err
-		}
-		err = Uncompress(src, g.folder)
-		if err != nil {
-			return err
-		}
-		// Gitlab zip/tar has contents inside a folder
-		// It has the format of eg. basic-micro-service-master-314b4a494ed472793e0a8bce8babbc69359aed7b
-		// Since we don't have the commit at this point we must list the dir
-		files, err := ioutil.ReadDir(g.folder)
-		if err != nil {
-			return err
-		}
-		if len(files) == 0 {
-			return fmt.Errorf("No contents in dir downloaded from gitlab: %v", g.folder)
-		}
-		g.folder = filepath.Join(g.folder, files[0].Name())
 		return nil
 	}
 	return fmt.Errorf("Repo host %v is not supported yet", repo)
 }
 
+func (g *binaryGitter) checkoutGithub(repo, branchOrCommit string) error {
+	// @todo if it's a commit it must not be checked out all the time
+	repoFolder := strings.ReplaceAll(strings.ReplaceAll(repo, "/", "-"), "https://", "")
+	g.folder = filepath.Join(os.TempDir(),
+		repoFolder+"-"+shortid.MustGenerate())
+
+	url := fmt.Sprintf("%v/archive/%v.zip", repo, branchOrCommit)
+	if !strings.HasPrefix(url, "https://") {
+		url = "https://" + url
+	}
+	client := &http.Client{}
+	req, _ := http.NewRequest("GET", url, nil)
+	if len(g.secrets["GIT_CREDENTIALS"]) > 0 {
+		req.Header.Set("Authorization", "token "+g.secrets["GIT_CREDENTIALS"])
+	}
+	resp, err := client.Do(req)
+	if err != nil {
+		return fmt.Errorf("Can't get zip: %v", err)
+	}
+
+	defer resp.Body.Close()
+	// Github returns 404 for tar.gz files...
+	// but still gives back a proper file so ignoring status code
+	// for now.
+	//if resp.StatusCode != 200 {
+	//	return errors.New("Status code was not 200")
+	//}
+
+	src := g.folder + ".zip"
+	// Create the file
+	out, err := os.Create(src)
+	if err != nil {
+		return fmt.Errorf("Can't create source file %v src: %v", src, err)
+	}
+	defer out.Close()
+
+	// Write the body to file
+	_, err = io.Copy(out, resp.Body)
+	if err != nil {
+		return err
+	}
+	return unzip(src, g.folder, true)
+}
+
+func (g *binaryGitter) checkoutGitLabPublic(repo, branchOrCommit string) error {
+	// Example: https://gitlab.com/micro-test/basic-micro-service/-/archive/master/basic-micro-service-master.tar.gz
+	// @todo if it's a commit it must not be checked out all the time
+	repoFolder := strings.ReplaceAll(strings.ReplaceAll(repo, "/", "-"), "https://", "")
+	g.folder = filepath.Join(os.TempDir(),
+		repoFolder+"-"+shortid.MustGenerate())
+
+	tarName := strings.ReplaceAll(strings.ReplaceAll(repo, "gitlab.com/", ""), "/", "-")
+	url := fmt.Sprintf("%v/-/archive/%v/%v.tar.gz", repo, branchOrCommit, tarName)
+	if !strings.HasPrefix(url, "https://") {
+		url = "https://" + url
+	}
+	client := &http.Client{}
+	req, _ := http.NewRequest("GET", url, nil)
+	resp, err := client.Do(req)
+	if err != nil {
+		return fmt.Errorf("Can't get zip: %v", err)
+	}
+
+	defer resp.Body.Close()
+
+	src := g.folder + ".tar.gz"
+	// Create the file
+	out, err := os.Create(src)
+	if err != nil {
+		return fmt.Errorf("Can't create source file %v src: %v", src, err)
+	}
+	defer out.Close()
+
+	// Write the body to file
+	_, err = io.Copy(out, resp.Body)
+	if err != nil {
+		return err
+	}
+	err = Uncompress(src, g.folder)
+	if err != nil {
+		return err
+	}
+	// Gitlab zip/tar has contents inside a folder
+	// It has the format of eg. basic-micro-service-master-314b4a494ed472793e0a8bce8babbc69359aed7b
+	// Since we don't have the commit at this point we must list the dir
+	files, err := ioutil.ReadDir(g.folder)
+	if err != nil {
+		return err
+	}
+	if len(files) == 0 {
+		return fmt.Errorf("No contents in dir downloaded from gitlab: %v", g.folder)
+	}
+	g.folder = filepath.Join(g.folder, files[0].Name())
+	return nil
+}
+
+func (g *binaryGitter) checkoutGitLabPrivate(repo, branchOrCommit string) error {
+	git, err := gitlab.NewClient(g.secrets["GIT_CREDENTIALS"])
+	if err != nil {
+		return err
+	}
+	owned := true
+	projects, _, err := git.Projects.ListProjects(&gitlab.ListProjectsOptions{
+		Owned: &owned,
+	})
+	if err != nil {
+		return err
+	}
+	projectID := ""
+	for _, project := range projects {
+		if strings.Contains(repo, project.Name) {
+			projectID = fmt.Sprintf("%v", project.ID)
+		}
+	}
+	if len(projectID) == 0 {
+		return fmt.Errorf("Project id not found for repo %v", repo)
+	}
+	// Example URL:
+	// https://gitlab.com/api/v3/projects/0000000/repository/archive?private_token=XXXXXXXXXXXXXXXXXXXX
+	url := fmt.Sprintf("https://gitlab.com/api/v4/projects/%v/repository/archive?private_token=%v", projectID, g.secrets["GIT_CREDENTIALS"])
+
+	client := &http.Client{}
+	req, _ := http.NewRequest("GET", url, nil)
+	resp, err := client.Do(req)
+	if err != nil {
+		return fmt.Errorf("Can't get zip: %v", err)
+	}
+
+	defer resp.Body.Close()
+
+	src := g.folder + ".tar.gz"
+	// Create the file
+	out, err := os.Create(src)
+	if err != nil {
+		return fmt.Errorf("Can't create source file %v src: %v", src, err)
+	}
+	defer out.Close()
+
+	// Write the body to file
+	_, err = io.Copy(out, resp.Body)
+	if err != nil {
+		return err
+	}
+	err = Uncompress(src, g.folder)
+	if err != nil {
+		return err
+	}
+	// Gitlab zip/tar has contents inside a folder
+	// It has the format of eg. basic-micro-service-master-314b4a494ed472793e0a8bce8babbc69359aed7b
+	// Since we don't have the commit at this point we must list the dir
+	files, err := ioutil.ReadDir(g.folder)
+	if err != nil {
+		return err
+	}
+	if len(files) == 0 {
+		return fmt.Errorf("No contents in dir downloaded from gitlab: %v", g.folder)
+	}
+	g.folder = filepath.Join(g.folder, files[0].Name())
+	return nil
+}
+
 func (g *binaryGitter) RepoDir() string {
 	return g.folder
 }
 
-func NewGitter(folder string) Gitter {
+func NewGitter(folder string, secrets map[string]string) Gitter {
-	return &binaryGitter{folder}
+	return &binaryGitter{folder, secrets}
 
 }
 
@@ -299,12 +388,12 @@ func IsLocal(workDir, source string, pathExistsFunc ...func(path string) (bool,
 // CheckoutSource for the local runtime server
 // folder is the folder to check out the source code to
 // Modifies source path to set it to checked out repo absolute path locally.
-func CheckoutSource(folder string, source *Source) error {
+func CheckoutSource(folder string, source *Source, secrets map[string]string) error {
 	// if it's a local folder, do nothing
 	if exists, err := pathExists(source.FullPath); err == nil && exists {
 		return nil
 	}
-	gitter := NewGitter(folder)
+	gitter := NewGitter(folder, secrets)
 	repo := source.Repo
 	if !strings.Contains(repo, "https://") {
 		repo = "https://" + repo

runtime/local/local.go

@@ -63,7 +63,7 @@ func NewRuntime(opts ...runtime.Option) runtime.Runtime {
 	}
 }
 
-func (r *localRuntime) checkoutSourceIfNeeded(s *runtime.Service) error {
+func (r *localRuntime) checkoutSourceIfNeeded(s *runtime.Service, secrets map[string]string) error {
 	// Runtime service like config have no source.
 	// Skip checkout in that case
 	if len(s.Source) == 0 {
@@ -109,7 +109,7 @@ func (r *localRuntime) checkoutSourceIfNeeded(s *runtime.Service) error {
 	}
 	source.Ref = s.Version
 
-	err = git.CheckoutSource(os.TempDir(), source)
+	err = git.CheckoutSource(os.TempDir(), source, secrets)
 	if err != nil {
 		return err
 	}
@@ -271,17 +271,17 @@ func serviceKey(s *runtime.Service) string {
 
 // Create creates a new service which is then started by runtime
 func (r *localRuntime) Create(s *runtime.Service, opts ...runtime.CreateOption) error {
-	err := r.checkoutSourceIfNeeded(s)
+	var options runtime.CreateOptions
+	for _, o := range opts {
+		o(&options)
+	}
+	err := r.checkoutSourceIfNeeded(s, options.Secrets)
 	if err != nil {
 		return err
 	}
 	r.Lock()
 	defer r.Unlock()
 
-	var options runtime.CreateOptions
-	for _, o := range opts {
-		o(&options)
-	}
 	if len(options.Namespace) == 0 {
 		options.Namespace = defaultNamespace
 	}
@@ -487,15 +487,15 @@ func (r *localRuntime) Update(s *runtime.Service, opts ...runtime.UpdateOption)
 	for _, o := range opts {
 		o(&options)
 	}
-	if len(options.Namespace) == 0 {
+	err := r.checkoutSourceIfNeeded(s, options.Secrets)
-		options.Namespace = defaultNamespace
-	}
-
-	err := r.checkoutSourceIfNeeded(s)
 	if err != nil {
 		return err
 	}
 
+	if len(options.Namespace) == 0 {
+		options.Namespace = defaultNamespace
+	}
+
 	r.Lock()
 	srvs, ok := r.namespaces[options.Namespace]
 	r.Unlock()

runtime/local/source/git/git.go

@@ -1,87 +0,0 @@
-// Package git provides a git source
-package git
-
-import (
-	"os"
-	"path/filepath"
-	"strings"
-
-	"github.com/go-git/go-git/v5"
-	"github.com/micro/go-micro/v3/runtime/local/source"
-)
-
-// Source retrieves source code
-// An empty struct can be used
-type Source struct {
-	Options source.Options
-}
-
-func (g *Source) Fetch(url string) (*source.Repository, error) {
-	purl := url
-
-	if parts := strings.Split(url, "://"); len(parts) > 1 {
-		purl = parts[len(parts)-1]
-	}
-
-	name := filepath.Base(url)
-	path := filepath.Join(g.Options.Path, purl)
-
-	_, err := git.PlainClone(path, false, &git.CloneOptions{
-		URL: url,
-	})
-	if err == nil {
-		return &source.Repository{
-			Name: name,
-			Path: path,
-			URL:  url,
-		}, nil
-	}
-
-	// repo already exists
-	if err != git.ErrRepositoryAlreadyExists {
-		return nil, err
-	}
-
-	// open repo
-	re, err := git.PlainOpen(path)
-	if err != nil {
-		return nil, err
-	}
-
-	// update it
-	if err := re.Fetch(nil); err != nil {
-		return nil, err
-	}
-
-	return &source.Repository{
-		Name: name,
-		Path: path,
-		URL:  url,
-	}, nil
-}
-
-func (g *Source) Commit(r *source.Repository) error {
-	repo := filepath.Join(r.Path)
-	re, err := git.PlainOpen(repo)
-	if err != nil {
-		return err
-	}
-	return re.Push(nil)
-}
-
-func (g *Source) String() string {
-	return "git"
-}
-
-func NewSource(opts ...source.Option) *Source {
-	options := source.Options{
-		Path: os.TempDir(),
-	}
-	for _, o := range opts {
-		o(&options)
-	}
-
-	return &Source{
-		Options: options,
-	}
-}

runtime/options.go

@@ -227,6 +227,19 @@ type UpdateOptions struct {
 	Namespace string
 	// Specify the context to use
 	Context context.Context
+	// Secrets to use
+	Secrets map[string]string
+}
+
+// WithSecret sets a secret to provide the service with
+func UpdateSecret(key, value string) UpdateOption {
+	return func(o *UpdateOptions) {
+		if o.Secrets == nil {
+			o.Secrets = map[string]string{key: value}
+		} else {
+			o.Secrets[key] = value
+		}
+	}
 }
 
 // UpdateNamespace sets the namespace