Support private repos in env 'local' (#1938)
This commit is contained in:
Janos Dobronszki
GitHub
@@ -16,6 +16,7 @@ import (
|
||||
"strings"
|
||||
|
||||
"github.com/teris-io/shortid"
|
||||
"github.com/xanzy/go-gitlab"
|
||||
)
|
||||
|
||||
type Gitter interface {
|
||||
@@ -24,7 +25,8 @@ type Gitter interface {
|
||||
}
|
||||
|
||||
type binaryGitter struct {
|
||||
folder string
|
||||
folder string
|
||||
secrets map[string]string
|
||||
}
|
||||
|
||||
func (g *binaryGitter) Checkout(repo, branchOrCommit string) error {
|
||||
@@ -38,100 +40,187 @@ func (g *binaryGitter) Checkout(repo, branchOrCommit string) error {
|
||||
branchOrCommit = "master"
|
||||
}
|
||||
if strings.Contains(repo, "github") {
|
||||
// @todo if it's a commit it must not be checked out all the time
|
||||
repoFolder := strings.ReplaceAll(strings.ReplaceAll(repo, "/", "-"), "https://", "")
|
||||
g.folder = filepath.Join(os.TempDir(),
|
||||
repoFolder+"-"+shortid.MustGenerate())
|
||||
|
||||
url := fmt.Sprintf("%v/archive/%v.zip", repo, branchOrCommit)
|
||||
if !strings.HasPrefix(url, "https://") {
|
||||
url = "https://" + url
|
||||
}
|
||||
resp, err := http.Get(url)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Can't get zip: %v", err)
|
||||
}
|
||||
|
||||
defer resp.Body.Close()
|
||||
// Github returns 404 for tar.gz files...
|
||||
// but still gives back a proper file so ignoring status code
|
||||
// for now.
|
||||
//if resp.StatusCode != 200 {
|
||||
// return errors.New("Status code was not 200")
|
||||
//}
|
||||
|
||||
src := g.folder + ".zip"
|
||||
// Create the file
|
||||
out, err := os.Create(src)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Can't create source file %v src: %v", src, err)
|
||||
}
|
||||
defer out.Close()
|
||||
|
||||
// Write the body to file
|
||||
_, err = io.Copy(out, resp.Body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return unzip(src, g.folder, true)
|
||||
return g.checkoutGithub(repo, branchOrCommit)
|
||||
} else if strings.Contains(repo, "gitlab") {
|
||||
// Example: https://gitlab.com/micro-test/basic-micro-service/-/archive/master/basic-micro-service-master.tar.gz
|
||||
// @todo if it's a commit it must not be checked out all the time
|
||||
repoFolder := strings.ReplaceAll(strings.ReplaceAll(repo, "/", "-"), "https://", "")
|
||||
g.folder = filepath.Join(os.TempDir(),
|
||||
repoFolder+"-"+shortid.MustGenerate())
|
||||
|
||||
tarName := strings.ReplaceAll(strings.ReplaceAll(repo, "gitlab.com/", ""), "/", "-")
|
||||
url := fmt.Sprintf("%v/-/archive/%v/%v.tar.gz", repo, branchOrCommit, tarName)
|
||||
if !strings.HasPrefix(url, "https://") {
|
||||
url = "https://" + url
|
||||
}
|
||||
resp, err := http.Get(url)
|
||||
err := g.checkoutGitLabPublic(repo, branchOrCommit)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Can't get zip: %v", err)
|
||||
// If the public download fails, try getting it with tokens.
|
||||
// Private downloads needs a token for api project listing, hence
|
||||
// the weird structure of this code.
|
||||
return g.checkoutGitLabPrivate(repo, branchOrCommit)
|
||||
}
|
||||
|
||||
defer resp.Body.Close()
|
||||
|
||||
src := g.folder + ".tar.gz"
|
||||
// Create the file
|
||||
out, err := os.Create(src)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Can't create source file %v src: %v", src, err)
|
||||
}
|
||||
defer out.Close()
|
||||
|
||||
// Write the body to file
|
||||
_, err = io.Copy(out, resp.Body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = Uncompress(src, g.folder)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// Gitlab zip/tar has contents inside a folder
|
||||
// It has the format of eg. basic-micro-service-master-314b4a494ed472793e0a8bce8babbc69359aed7b
|
||||
// Since we don't have the commit at this point we must list the dir
|
||||
files, err := ioutil.ReadDir(g.folder)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(files) == 0 {
|
||||
return fmt.Errorf("No contents in dir downloaded from gitlab: %v", g.folder)
|
||||
}
|
||||
g.folder = filepath.Join(g.folder, files[0].Name())
|
||||
return nil
|
||||
}
|
||||
return fmt.Errorf("Repo host %v is not supported yet", repo)
|
||||
}
|
||||
|
||||
func (g *binaryGitter) checkoutGithub(repo, branchOrCommit string) error {
|
||||
// @todo if it's a commit it must not be checked out all the time
|
||||
repoFolder := strings.ReplaceAll(strings.ReplaceAll(repo, "/", "-"), "https://", "")
|
||||
g.folder = filepath.Join(os.TempDir(),
|
||||
repoFolder+"-"+shortid.MustGenerate())
|
||||
|
||||
url := fmt.Sprintf("%v/archive/%v.zip", repo, branchOrCommit)
|
||||
if !strings.HasPrefix(url, "https://") {
|
||||
url = "https://" + url
|
||||
}
|
||||
client := &http.Client{}
|
||||
req, _ := http.NewRequest("GET", url, nil)
|
||||
if len(g.secrets["GIT_CREDENTIALS"]) > 0 {
|
||||
req.Header.Set("Authorization", "token "+g.secrets["GIT_CREDENTIALS"])
|
||||
}
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Can't get zip: %v", err)
|
||||
}
|
||||
|
||||
defer resp.Body.Close()
|
||||
// Github returns 404 for tar.gz files...
|
||||
// but still gives back a proper file so ignoring status code
|
||||
// for now.
|
||||
//if resp.StatusCode != 200 {
|
||||
// return errors.New("Status code was not 200")
|
||||
//}
|
||||
|
||||
src := g.folder + ".zip"
|
||||
// Create the file
|
||||
out, err := os.Create(src)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Can't create source file %v src: %v", src, err)
|
||||
}
|
||||
defer out.Close()
|
||||
|
||||
// Write the body to file
|
||||
_, err = io.Copy(out, resp.Body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return unzip(src, g.folder, true)
|
||||
}
|
||||
|
||||
func (g *binaryGitter) checkoutGitLabPublic(repo, branchOrCommit string) error {
|
||||
// Example: https://gitlab.com/micro-test/basic-micro-service/-/archive/master/basic-micro-service-master.tar.gz
|
||||
// @todo if it's a commit it must not be checked out all the time
|
||||
repoFolder := strings.ReplaceAll(strings.ReplaceAll(repo, "/", "-"), "https://", "")
|
||||
g.folder = filepath.Join(os.TempDir(),
|
||||
repoFolder+"-"+shortid.MustGenerate())
|
||||
|
||||
tarName := strings.ReplaceAll(strings.ReplaceAll(repo, "gitlab.com/", ""), "/", "-")
|
||||
url := fmt.Sprintf("%v/-/archive/%v/%v.tar.gz", repo, branchOrCommit, tarName)
|
||||
if !strings.HasPrefix(url, "https://") {
|
||||
url = "https://" + url
|
||||
}
|
||||
client := &http.Client{}
|
||||
req, _ := http.NewRequest("GET", url, nil)
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Can't get zip: %v", err)
|
||||
}
|
||||
|
||||
defer resp.Body.Close()
|
||||
|
||||
src := g.folder + ".tar.gz"
|
||||
// Create the file
|
||||
out, err := os.Create(src)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Can't create source file %v src: %v", src, err)
|
||||
}
|
||||
defer out.Close()
|
||||
|
||||
// Write the body to file
|
||||
_, err = io.Copy(out, resp.Body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = Uncompress(src, g.folder)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// Gitlab zip/tar has contents inside a folder
|
||||
// It has the format of eg. basic-micro-service-master-314b4a494ed472793e0a8bce8babbc69359aed7b
|
||||
// Since we don't have the commit at this point we must list the dir
|
||||
files, err := ioutil.ReadDir(g.folder)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(files) == 0 {
|
||||
return fmt.Errorf("No contents in dir downloaded from gitlab: %v", g.folder)
|
||||
}
|
||||
g.folder = filepath.Join(g.folder, files[0].Name())
|
||||
return nil
|
||||
}
|
||||
|
||||
func (g *binaryGitter) checkoutGitLabPrivate(repo, branchOrCommit string) error {
|
||||
git, err := gitlab.NewClient(g.secrets["GIT_CREDENTIALS"])
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
owned := true
|
||||
projects, _, err := git.Projects.ListProjects(&gitlab.ListProjectsOptions{
|
||||
Owned: &owned,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
projectID := ""
|
||||
for _, project := range projects {
|
||||
if strings.Contains(repo, project.Name) {
|
||||
projectID = fmt.Sprintf("%v", project.ID)
|
||||
}
|
||||
}
|
||||
if len(projectID) == 0 {
|
||||
return fmt.Errorf("Project id not found for repo %v", repo)
|
||||
}
|
||||
// Example URL:
|
||||
// https://gitlab.com/api/v3/projects/0000000/repository/archive?private_token=XXXXXXXXXXXXXXXXXXXX
|
||||
url := fmt.Sprintf("https://gitlab.com/api/v4/projects/%v/repository/archive?private_token=%v", projectID, g.secrets["GIT_CREDENTIALS"])
|
||||
|
||||
client := &http.Client{}
|
||||
req, _ := http.NewRequest("GET", url, nil)
|
||||
resp, err := client.Do(req)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Can't get zip: %v", err)
|
||||
}
|
||||
|
||||
defer resp.Body.Close()
|
||||
|
||||
src := g.folder + ".tar.gz"
|
||||
// Create the file
|
||||
out, err := os.Create(src)
|
||||
if err != nil {
|
||||
return fmt.Errorf("Can't create source file %v src: %v", src, err)
|
||||
}
|
||||
defer out.Close()
|
||||
|
||||
// Write the body to file
|
||||
_, err = io.Copy(out, resp.Body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
err = Uncompress(src, g.folder)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
// Gitlab zip/tar has contents inside a folder
|
||||
// It has the format of eg. basic-micro-service-master-314b4a494ed472793e0a8bce8babbc69359aed7b
|
||||
// Since we don't have the commit at this point we must list the dir
|
||||
files, err := ioutil.ReadDir(g.folder)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if len(files) == 0 {
|
||||
return fmt.Errorf("No contents in dir downloaded from gitlab: %v", g.folder)
|
||||
}
|
||||
g.folder = filepath.Join(g.folder, files[0].Name())
|
||||
return nil
|
||||
}
|
||||
|
||||
func (g *binaryGitter) RepoDir() string {
|
||||
return g.folder
|
||||
}
|
||||
|
||||
func NewGitter(folder string) Gitter {
|
||||
return &binaryGitter{folder}
|
||||
func NewGitter(folder string, secrets map[string]string) Gitter {
|
||||
return &binaryGitter{folder, secrets}
|
||||
|
||||
}
|
||||
|
||||
@@ -299,12 +388,12 @@ func IsLocal(workDir, source string, pathExistsFunc ...func(path string) (bool,
|
||||
// CheckoutSource for the local runtime server
|
||||
// folder is the folder to check out the source code to
|
||||
// Modifies source path to set it to checked out repo absolute path locally.
|
||||
func CheckoutSource(folder string, source *Source) error {
|
||||
func CheckoutSource(folder string, source *Source, secrets map[string]string) error {
|
||||
// if it's a local folder, do nothing
|
||||
if exists, err := pathExists(source.FullPath); err == nil && exists {
|
||||
return nil
|
||||
}
|
||||
gitter := NewGitter(folder)
|
||||
gitter := NewGitter(folder, secrets)
|
||||
repo := source.Repo
|
||||
if !strings.Contains(repo, "https://") {
|
||||
repo = "https://" + repo
|
||||
|
||||
@@ -63,7 +63,7 @@ func NewRuntime(opts ...runtime.Option) runtime.Runtime {
|
||||
}
|
||||
}
|
||||
|
||||
func (r *localRuntime) checkoutSourceIfNeeded(s *runtime.Service) error {
|
||||
func (r *localRuntime) checkoutSourceIfNeeded(s *runtime.Service, secrets map[string]string) error {
|
||||
// Runtime service like config have no source.
|
||||
// Skip checkout in that case
|
||||
if len(s.Source) == 0 {
|
||||
@@ -109,7 +109,7 @@ func (r *localRuntime) checkoutSourceIfNeeded(s *runtime.Service) error {
|
||||
}
|
||||
source.Ref = s.Version
|
||||
|
||||
err = git.CheckoutSource(os.TempDir(), source)
|
||||
err = git.CheckoutSource(os.TempDir(), source, secrets)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@@ -271,17 +271,17 @@ func serviceKey(s *runtime.Service) string {
|
||||
|
||||
// Create creates a new service which is then started by runtime
|
||||
func (r *localRuntime) Create(s *runtime.Service, opts ...runtime.CreateOption) error {
|
||||
err := r.checkoutSourceIfNeeded(s)
|
||||
var options runtime.CreateOptions
|
||||
for _, o := range opts {
|
||||
o(&options)
|
||||
}
|
||||
err := r.checkoutSourceIfNeeded(s, options.Secrets)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
r.Lock()
|
||||
defer r.Unlock()
|
||||
|
||||
var options runtime.CreateOptions
|
||||
for _, o := range opts {
|
||||
o(&options)
|
||||
}
|
||||
if len(options.Namespace) == 0 {
|
||||
options.Namespace = defaultNamespace
|
||||
}
|
||||
@@ -487,15 +487,15 @@ func (r *localRuntime) Update(s *runtime.Service, opts ...runtime.UpdateOption)
|
||||
for _, o := range opts {
|
||||
o(&options)
|
||||
}
|
||||
if len(options.Namespace) == 0 {
|
||||
options.Namespace = defaultNamespace
|
||||
}
|
||||
|
||||
err := r.checkoutSourceIfNeeded(s)
|
||||
err := r.checkoutSourceIfNeeded(s, options.Secrets)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if len(options.Namespace) == 0 {
|
||||
options.Namespace = defaultNamespace
|
||||
}
|
||||
|
||||
r.Lock()
|
||||
srvs, ok := r.namespaces[options.Namespace]
|
||||
r.Unlock()
|
||||
|
||||
@@ -1,87 +0,0 @@
|
||||
// Package git provides a git source
|
||||
package git
|
||||
|
||||
import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"github.com/go-git/go-git/v5"
|
||||
"github.com/micro/go-micro/v3/runtime/local/source"
|
||||
)
|
||||
|
||||
// Source retrieves source code
|
||||
// An empty struct can be used
|
||||
type Source struct {
|
||||
Options source.Options
|
||||
}
|
||||
|
||||
func (g *Source) Fetch(url string) (*source.Repository, error) {
|
||||
purl := url
|
||||
|
||||
if parts := strings.Split(url, "://"); len(parts) > 1 {
|
||||
purl = parts[len(parts)-1]
|
||||
}
|
||||
|
||||
name := filepath.Base(url)
|
||||
path := filepath.Join(g.Options.Path, purl)
|
||||
|
||||
_, err := git.PlainClone(path, false, &git.CloneOptions{
|
||||
URL: url,
|
||||
})
|
||||
if err == nil {
|
||||
return &source.Repository{
|
||||
Name: name,
|
||||
Path: path,
|
||||
URL: url,
|
||||
}, nil
|
||||
}
|
||||
|
||||
// repo already exists
|
||||
if err != git.ErrRepositoryAlreadyExists {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// open repo
|
||||
re, err := git.PlainOpen(path)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// update it
|
||||
if err := re.Fetch(nil); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return &source.Repository{
|
||||
Name: name,
|
||||
Path: path,
|
||||
URL: url,
|
||||
}, nil
|
||||
}
|
||||
|
||||
func (g *Source) Commit(r *source.Repository) error {
|
||||
repo := filepath.Join(r.Path)
|
||||
re, err := git.PlainOpen(repo)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
return re.Push(nil)
|
||||
}
|
||||
|
||||
func (g *Source) String() string {
|
||||
return "git"
|
||||
}
|
||||
|
||||
func NewSource(opts ...source.Option) *Source {
|
||||
options := source.Options{
|
||||
Path: os.TempDir(),
|
||||
}
|
||||
for _, o := range opts {
|
||||
o(&options)
|
||||
}
|
||||
|
||||
return &Source{
|
||||
Options: options,
|
||||
}
|
||||
}
|
||||
@@ -227,6 +227,19 @@ type UpdateOptions struct {
|
||||
Namespace string
|
||||
// Specify the context to use
|
||||
Context context.Context
|
||||
// Secrets to use
|
||||
Secrets map[string]string
|
||||
}
|
||||
|
||||
// WithSecret sets a secret to provide the service with
|
||||
func UpdateSecret(key, value string) UpdateOption {
|
||||
return func(o *UpdateOptions) {
|
||||
if o.Secrets == nil {
|
||||
o.Secrets = map[string]string{key: value}
|
||||
} else {
|
||||
o.Secrets[key] = value
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// UpdateNamespace sets the namespace
|
||||
|
||||
Reference in New Issue
Block a user