Signed-off-by: Vasiliy Tolstov <v.tolstov@unistack.org>
This commit is contained in:
2024-12-07 02:35:30 +03:00
parent 8729d0b88e
commit c5f3fa325e
42 changed files with 1316 additions and 2009 deletions

View File

@@ -17,23 +17,22 @@ import (
"text/template"
"time"
"git.unistack.org/unistack-org/pkgdash/internal/configcli"
"git.unistack.org/unistack-org/pkgdash/internal/modules"
"git.unistack.org/unistack-org/pkgdash/internal/source"
"github.com/go-git/go-git/v5"
gitconfig "github.com/go-git/go-git/v5/config"
"github.com/go-git/go-git/v5/plumbing"
"github.com/go-git/go-git/v5/plumbing/object"
httpauth "github.com/go-git/go-git/v5/plumbing/transport/http"
"github.com/jdx/go-netrc"
yamlcodec "go.unistack.org/micro-codec-yaml/v4"
envconfig "go.unistack.org/micro-config-env/v4"
fileconfig "go.unistack.org/micro-config-file/v4"
microflag "go.unistack.org/micro-config-flag/v4"
"go.unistack.org/micro/v4/config"
"go.unistack.org/micro/v4/logger"
"go.unistack.org/micro/v4/logger/slog"
"go.unistack.org/micro/v4/options"
yamlcodec "go.unistack.org/micro-codec-yaml/v3"
envconfig "go.unistack.org/micro-config-env/v3"
fileconfig "go.unistack.org/micro-config-file/v3"
microflag "go.unistack.org/micro-config-flag/v3"
"go.unistack.org/micro/v3/config"
"go.unistack.org/micro/v3/logger"
"go.unistack.org/micro/v3/logger/slog"
"go.unistack.org/pkgdash/internal/configcli"
"go.unistack.org/pkgdash/internal/modules"
"go.unistack.org/pkgdash/internal/source"
"golang.org/x/mod/modfile"
"golang.org/x/mod/semver"
)
@@ -98,10 +97,10 @@ func main() {
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
logger.DefaultLogger = slog.NewLogger()
log := slog.NewLogger()
if err = logger.DefaultLogger.Init(logger.WithCallerSkipCount(3), logger.WithLevel(logger.DebugLevel)); err != nil {
logger.Error(ctx, fmt.Sprintf("logger init error: %v", err))
if err = log.Init(logger.WithLevel(logger.DebugLevel)); err != nil {
log.Error(ctx, fmt.Sprintf("logger init error: %v", err))
}
cfg := configcli.NewConfig()
@@ -117,7 +116,7 @@ func main() {
},
config.LoadOverride(true),
); err != nil {
logger.Fatal(ctx, fmt.Sprintf("failed to load config: %v", err))
log.Fatal(ctx, fmt.Sprintf("failed to load config: %v", err))
}
for _, configDir := range configDirs {
@@ -130,15 +129,15 @@ func main() {
c := fileconfig.NewConfig(
config.AllowFail(false),
config.Struct(cfg),
options.Codec(yamlcodec.NewCodec()),
config.Codec(yamlcodec.NewCodec()),
fileconfig.Path(path),
)
err = c.Init()
if err != nil {
logger.Error(ctx, fmt.Sprintf("failed to init config: %v", err))
log.Error(ctx, fmt.Sprintf("failed to init config: %v", err))
}
if err = c.Load(ctx, config.LoadOverride(true)); err != nil {
logger.Error(ctx, fmt.Sprintf("failed to load config: %v", err))
log.Error(ctx, fmt.Sprintf("failed to load config: %v", err))
}
}
}
@@ -155,11 +154,11 @@ func main() {
c := microflag.NewConfig(config.Struct(cliCfg), microflag.FlagErrorHandling(flag.ContinueOnError))
if err = c.Init(); err != nil {
logger.Fatal(ctx, fmt.Sprintf("init cli cfg failed: %v", err))
log.Fatal(ctx, fmt.Sprintf("init cli cfg failed: %v", err))
}
if err = c.Load(ctx); err != nil {
logger.Fatal(ctx, fmt.Sprintf("load cli cfg failed: %v", err))
log.Fatal(ctx, fmt.Sprintf("load cli cfg failed: %v", err))
}
if cliCfg.Path == "" && cliCfg.Command == "" {
@@ -178,11 +177,11 @@ func main() {
}
buf, err := os.ReadFile(name)
if err != nil {
panic(err)
log.Fatal(ctx, "failed to read file", err)
}
mfile, err := modfile.Parse(name, buf, nil)
if err != nil {
panic(err)
log.Fatal(ctx, "failed to parse file", err)
}
mvs := make(map[string]modules.Update)
@@ -195,7 +194,7 @@ func main() {
OnUpdate: func(u modules.Update) {
var modpath string // new mod path with major
if u.Err != nil {
logger.Error(ctx, fmt.Sprintf("%s: failed: %v", u.Module.Path, u.Err))
log.Error(ctx, fmt.Sprintf("%s: failed: %v", u.Module.Path, u.Err))
return
}
modpath = u.Module.Path
@@ -219,18 +218,34 @@ func main() {
modules.Updates(updateOptions)
if err = getRepoMgmt(ctx, cfg); err != nil { // Filling in empty config fields.
logger.Error(ctx, err.Error())
if err = getRepoMgmt(ctx, log, cfg); err != nil { // Filling in empty config fields.
log.Error(ctx, err.Error())
}
gitSource := source.NewSourceControl(*cfg)
if len(cfg.Branches) == 0 {
branchName, err := getCurrentBranch(ctx)
if err != nil {
log.Fatal(ctx, "failed to get current branch", err)
}
cfg.Branches = append(cfg.Branches, branchName)
}
Execute(ctx, gitSource, mvs, *cliCfg, *cfg)
if cfg.Source.Repository == "" {
reposiotry, err := getCurrentRepository(ctx)
if err != nil {
log.Fatal(ctx, "failed to get current repository", err)
}
cfg.Source.Repository = reposiotry
}
logger.Info(ctx, "Pkgdash successfully updated dependencies")
gitSource := source.NewSourceControl(*cfg, log)
Execute(ctx, log, gitSource, mvs, *cliCfg, *cfg)
log.Info(ctx, "Pkgdash successfully updated dependencies")
}
func Execute(ctx context.Context, gitSource source.SourceControl, mvs map[string]modules.Update, cliCfg configcli.Cli, cfg configcli.Config) {
func Execute(ctx context.Context, log logger.Logger, gitSource source.SourceControl, mvs map[string]modules.Update, cliCfg configcli.Cli, cfg configcli.Config) {
var mod modules.Update
var ok bool
var path string
@@ -244,81 +259,131 @@ func Execute(ctx context.Context, gitSource source.SourceControl, mvs map[string
if cliCfg.Path != "" { // update one dep
path = cliCfg.Path
if mod, ok = mvs[path]; !ok {
logger.Fatal(ctx, fmt.Sprintf("For %s update not exist", path))
log.Fatal(ctx, fmt.Sprintf("For %s update not exist", path))
}
logger.Debug(ctx, fmt.Sprintf("Start update %s from %s to %s", path, mod.Module.Version, mod.Version))
log.Debug(ctx, fmt.Sprintf("Start update %s from %s to %s", path, mod.Module.Version, mod.Version))
for _, branch := range cfg.Branches {
if err := gitSource.RequestOpen(ctx, branch, path, mod); err != nil {
logger.Fatal(ctx, fmt.Sprintf("failed to create pr: %v", err))
log.Fatal(ctx, fmt.Sprintf("failed to create pr: %v", err))
}
}
logger.Debug(ctx, fmt.Sprintf("Update successful for %s", path))
log.Debug(ctx, fmt.Sprintf("Update successful for %s", path))
return
}
for _, branch := range cfg.Branches { // update all dep
for path, mod = range mvs {
logger.Debug(ctx, fmt.Sprintf("Start update %s from %s to %s", path, mod.Module.Version, mod.Version))
log.Debug(ctx, fmt.Sprintf("Start update %s from %s to %s", path, mod.Module.Version, mod.Version))
err := gitSource.RequestOpen(ctx, branch, path, mod)
if err != nil {
if strings.Contains(err.Error(), "already exists") {
logger.Debug(ctx, fmt.Sprintf("skip %s, branch already exists", path))
log.Debug(ctx, fmt.Sprintf("skip %s, branch already exists", path))
continue
}
logger.Fatal(ctx, fmt.Sprintf("failed to create pr: %v", err))
log.Fatal(ctx, fmt.Sprintf("failed to create pr: %v", err))
}
logger.Debug(ctx, fmt.Sprintf("Update successful for %s", path))
log.Debug(ctx, fmt.Sprintf("Update successful for %s", path))
}
}
case "close":
if cliCfg.Path != "" { // close one dep
path = cliCfg.Path
logger.Debug(ctx, fmt.Sprintf("Start close for %s", path))
log.Debug(ctx, fmt.Sprintf("Start close for %s", path))
for _, branch := range cfg.Branches {
if err := gitSource.RequestClose(ctx, branch, path); err != nil {
logger.Fatal(ctx, fmt.Sprintf("failed to close pr: %v", err))
log.Fatal(ctx, fmt.Sprintf("failed to close pr: %v", err))
}
}
logger.Debug(ctx, fmt.Sprintf("Close successful for %s", path))
log.Debug(ctx, fmt.Sprintf("Close successful for %s", path))
return
}
for _, branch := range cfg.Branches {
logger.Info(ctx, fmt.Sprintf("Start getting pr for %s", branch))
log.Info(ctx, fmt.Sprintf("Start getting pr for %s", branch))
rMap, err := gitSource.RequestList(ctx, branch)
if err != nil {
logger.Fatal(ctx, fmt.Sprintf("Error with getting pr list for branch: %s", branch))
log.Fatal(ctx, fmt.Sprintf("Error with getting pr list for branch: %s", branch))
}
logger.Info(ctx, fmt.Sprintf("for %s:\n%s", branch, rMap))
logger.Info(ctx, fmt.Sprintf("Start close pr for base branch %s", branch))
log.Info(ctx, fmt.Sprintf("for %s:\n%s", branch, rMap))
log.Info(ctx, fmt.Sprintf("Start close pr for base branch %s", branch))
for path, _ = range rMap {
logger.Debug(ctx, fmt.Sprintf("Start close for %s", path))
for path = range rMap {
log.Debug(ctx, fmt.Sprintf("Start close for %s", path))
if err = gitSource.RequestClose(ctx, branch, path); err != nil {
logger.Fatal(ctx, fmt.Sprintf("failed to close pr: %v", err))
log.Fatal(ctx, fmt.Sprintf("failed to close pr: %v", err))
}
logger.Debug(ctx, fmt.Sprintf("Close successful for %s", path))
log.Debug(ctx, fmt.Sprintf("Close successful for %s", path))
}
}
case "list":
for _, branch := range cfg.Branches {
rMap, err := gitSource.RequestList(ctx, branch)
if err != nil {
logger.Fatal(ctx, fmt.Sprintf("RequestList: error %s", err))
log.Fatal(ctx, fmt.Sprintf("RequestList: error %s", err))
}
prList[branch] = rMap
}
js, err := json.Marshal(prList)
if err != nil {
logger.Error(ctx, fmt.Sprintf("error: %s", err))
log.Error(ctx, fmt.Sprintf("error: %s", err))
}
fmt.Println(fmt.Sprintf("for %s:\n%s", cfg.Source.Repository, js))
fmt.Printf("for %s:\n%s\n", cfg.Source.Repository, js)
default:
fmt.Print(initMsg)
}
}
func getRepoMgmt(ctx context.Context, cfg *configcli.Config) error {
func getCurrentRepository(ctx context.Context) (string, error) {
wd, err := os.Getwd()
if err != nil {
return "", err
}
p := filepath.Clean(wd)
repo, err := git.PlainOpen(p)
if err != nil {
return "", err
}
cfg, err := repo.Config()
if err != nil {
return "", err
}
for k, v := range cfg.Remotes {
if k != "origin" {
continue
}
return v.URLs[0], nil
}
return "", fmt.Errorf("failed to get remotes")
}
func getCurrentBranch(ctx context.Context) (string, error) {
wd, err := os.Getwd()
if err != nil {
return "", err
}
p := filepath.Clean(wd)
repo, err := git.PlainOpen(p)
if err != nil {
return "", err
}
ref, err := repo.Head()
if err != nil {
return "", err
}
return ref.Name().Short(), nil
}
func getRepoMgmt(ctx context.Context, log logger.Logger, cfg *configcli.Config) error {
wd, err := os.Getwd()
if err != nil {
return err
@@ -327,27 +392,38 @@ func getRepoMgmt(ctx context.Context, cfg *configcli.Config) error {
p := filepath.Clean(wd)
for _, configDir := range configDirs {
_, err := os.Stat(filepath.Join(p, configDir))
if name, ok := repoMgmt[configDir]; ok && cfg.Source.TypeGit == "" && err == nil {
if err != nil {
continue
}
log.Info(ctx, fmt.Sprintf("check config dir %s", configDir))
if name, ok := repoMgmt[configDir]; ok && cfg.Source.TypeGit == "" {
cfg.Source.TypeGit = name
}
if api, ok := repoAPI[configDir]; ok && cfg.Source.APIURL == "" && err == nil {
if api, ok := repoAPI[configDir]; ok && cfg.Source.APIURL == "" {
cfg.Source.APIURL = api
}
}
if p == "/" && cfg.Source.TypeGit == "" && cfg.Source.APIURL == "" {
return fmt.Errorf("unknown")
}
p = filepath.Clean(filepath.Join(p, ".."))
// p = filepath.Clean(filepath.Join(p, ".."))
usr, err := user.Current()
if err != nil {
logger.Fatal(ctx, fmt.Sprintf("pkgdash/main can t get info about user: %s", err))
}
n, err := netrc.Parse(filepath.Join(usr.HomeDir, ".netrc"))
if err != nil {
logger.Error(ctx, "pkgdash/main can t parse .netrc: %s", err)
log.Fatal(ctx, fmt.Sprintf("pkgdash/main cant get info about user: %s", err))
}
log.Info(ctx, fmt.Sprintf("try to configure scm source %v", cfg.Source))
netrcfile := filepath.Join(usr.HomeDir, ".netrc")
log.Info(ctx, "try to parse netrc file "+netrcfile)
n, err := netrc.Parse(netrcfile)
if err != nil {
log.Error(ctx, "pkgdash/main cant parse .netrc: %s", err)
}
log.Info(ctx, "try to configure scm for "+cfg.Source.APIURL)
if cfg.Source.Owner == "" {
cfg.Source.Owner = n.Machine(cfg.Source.APIURL).Get("login")
}
@@ -360,7 +436,7 @@ func getRepoMgmt(ctx context.Context, cfg *configcli.Config) error {
return nil
}
func giteaPullRequest(ctx context.Context, cfg *configcli.Config, branch string, mods map[string]modules.Update) error {
func giteaPullRequest(ctx context.Context, log logger.Logger, cfg *configcli.Config, branch string, mods map[string]modules.Update) error {
envAPIURL := os.Getenv("GITHUB_API_URL")
envREPOSITORY := os.Getenv("GITHUB_REPOSITORY")
envTOKEN := os.Getenv("GITHUB_TOKEN")
@@ -370,34 +446,34 @@ func giteaPullRequest(ctx context.Context, cfg *configcli.Config, branch string,
tplTitle, err := template.New("pull_request_title").Parse(cfg.PullRequestTitle)
if err != nil {
logger.Fatal(ctx, "failed to parse template: %v", err)
log.Fatal(ctx, "failed to parse template: %v", err)
}
wTitle := bytes.NewBuffer(nil)
tplBody, err := template.New("pull_request_body").Parse(cfg.PullRequestBody)
if err != nil {
logger.Fatal(ctx, "failed to parse template: %v", err)
log.Fatal(ctx, "failed to parse template: %v", err)
}
wBody := bytes.NewBuffer(nil)
repo, err := git.PlainOpenWithOptions(".", &git.PlainOpenOptions{DetectDotGit: true})
if err != nil {
logger.Fatal(ctx, "failed to open repo: %v", err)
log.Fatal(ctx, "failed to open repo: %v", err)
}
if err = repo.FetchContext(ctx, &git.FetchOptions{
Auth: &httpauth.BasicAuth{Username: envTOKEN, Password: envTOKEN},
Force: true,
}); err != nil && err != git.NoErrAlreadyUpToDate {
logger.Fatal(ctx, "failed to fetch repo: %v", err)
log.Fatal(ctx, "failed to fetch repo: %v", err)
}
var headRef *plumbing.Reference
refIter, err := repo.Branches()
if err != nil {
logger.Fatal(ctx, "failed to get branches: %v", err)
log.Fatal(ctx, "failed to get branches: %v", err)
}
for {
ref, err := refIter.Next()
@@ -412,14 +488,14 @@ func giteaPullRequest(ctx context.Context, cfg *configcli.Config, branch string,
refIter.Close()
if headRef == nil {
logger.Fatal(ctx, "failed to get repo branch head")
log.Fatal(ctx, "failed to get repo branch head")
}
logger.Info(ctx, "repo head %s", headRef)
log.Info(ctx, "repo head %s", headRef)
wtree, err := repo.Worktree()
if err != nil {
logger.Fatal(ctx, "failed to get worktree: %v", err)
log.Fatal(ctx, "failed to get worktree: %v", err)
}
type giteaPull struct {
@@ -450,13 +526,13 @@ func giteaPullRequest(ctx context.Context, cfg *configcli.Config, branch string,
}
if err = json.Unmarshal(buf, &pulls); err != nil {
logger.Fatal(ctx, "failed to decode response %s err: %v", buf, err)
log.Fatal(ctx, "failed to decode response %s err: %v", buf, err)
}
for path := range mods {
for _, pull := range pulls {
if strings.Contains(pull.Title, path) && pull.Base.Ref == branch {
logger.Info(ctx, "skip %s as pr already exists %s", path, pull.URL)
log.Info(ctx, "skip %s as pr already exists %s", path, pull.URL)
delete(mods, path)
}
}
@@ -466,11 +542,11 @@ func giteaPullRequest(ctx context.Context, cfg *configcli.Config, branch string,
wTitle.Reset()
wBody.Reset()
logger.Info(ctx, "update %s from %s to %s", path, mod.Module.Version, mod.Version)
log.Info(ctx, "update %s from %s to %s", path, mod.Module.Version, mod.Version)
logger.Info(ctx, "reset worktree")
log.Info(ctx, "reset worktree")
if err = wtree.Reset(&git.ResetOptions{Mode: git.HardReset}); err != nil {
logger.Fatal(ctx, "failed to reset repo branch: %v", err)
log.Fatal(ctx, "failed to reset repo branch: %v", err)
}
if err = wtree.PullContext(ctx, &git.PullOptions{
@@ -480,17 +556,17 @@ func giteaPullRequest(ctx context.Context, cfg *configcli.Config, branch string,
Force: true,
RemoteName: "origin",
}); err != nil && err != git.NoErrAlreadyUpToDate {
logger.Fatal(ctx, "failed to pull repo: %v", err)
log.Fatal(ctx, "failed to pull repo: %v", err)
}
logger.Info(ctx, "checkout ref %s", headRef)
log.Info(ctx, "checkout ref %s", headRef)
if err = wtree.Checkout(&git.CheckoutOptions{
Hash: headRef.Hash(),
Branch: plumbing.NewBranchReferenceName(fmt.Sprintf("pkgdash/go_modules/%s-%s", path, mod.Version)),
Create: true,
Force: true,
}); err != nil {
logger.Fatal(ctx, "failed to checkout tree: %v", err)
log.Fatal(ctx, "failed to checkout tree: %v", err)
}
epath, err := exec.LookPath("go")
@@ -498,7 +574,7 @@ func giteaPullRequest(ctx context.Context, cfg *configcli.Config, branch string,
err = nil
}
if err != nil {
logger.Fatal(ctx, "failed to find go command: %v", err)
log.Fatal(ctx, "failed to find go command: %v", err)
}
var cmd *exec.Cmd
@@ -506,25 +582,25 @@ func giteaPullRequest(ctx context.Context, cfg *configcli.Config, branch string,
cmd = exec.CommandContext(ctx, epath, "mod", "edit", fmt.Sprintf("-require=%s@%s", path, mod.Version))
if out, err = cmd.CombinedOutput(); err != nil {
logger.Fatal(ctx, "failed to run go mod edit: %s err: %v", out, err)
log.Fatal(ctx, "failed to run go mod edit: %s err: %v", out, err)
}
cmd = exec.CommandContext(ctx, epath, "mod", "tidy")
if out, err = cmd.CombinedOutput(); err != nil {
logger.Fatal(ctx, "failed to run go mod tidy: %s err: %v", out, err)
log.Fatal(ctx, "failed to run go mod tidy: %s err: %v", out, err)
}
logger.Info(ctx, "worktree add go.mod")
log.Info(ctx, "worktree add go.mod")
if _, err = wtree.Add("go.mod"); err != nil {
logger.Fatal(ctx, "failed to add file: %v", err)
log.Fatal(ctx, "failed to add file: %v", err)
}
logger.Info(ctx, "worktree add go.sum")
log.Info(ctx, "worktree add go.sum")
if _, err = wtree.Add("go.sum"); err != nil {
logger.Fatal(ctx, "failed to add file: %v", err)
log.Fatal(ctx, "failed to add file: %v", err)
}
logger.Info(ctx, "worktree commit")
log.Info(ctx, "worktree commit")
_, err = wtree.Commit(wTitle.String(), &git.CommitOptions{
Parents: []plumbing.Hash{headRef.Hash()},
Author: &object.Signature{
@@ -534,27 +610,27 @@ func giteaPullRequest(ctx context.Context, cfg *configcli.Config, branch string,
},
})
if err != nil {
logger.Fatal(ctx, "failed to commit: %v", err)
log.Fatal(ctx, "failed to commit: %v", err)
}
// newref := plumbing.NewHashReference(plumbing.ReferenceName(fmt.Sprintf("refs/heads/pkgdash/go_modules/%s-%s", path, mod.Version)), headRef.Hash())
/*
if err = repo.Storer.SetReference(newref); err != nil {
logger.Fatal(ctx, "failed to create repo branch: %v", err)
log.Fatal(ctx, "failed to create repo branch: %v", err)
}
*/
refspec := gitconfig.RefSpec(fmt.Sprintf("+refs/heads/pkgdash/go_modules/%s-%s:refs/heads/pkgdash/go_modules/%s-%s", path, mod.Version, path, mod.Version))
logger.Info(ctx, "try to push refspec %s", refspec)
log.Info(ctx, "try to push refspec %s", refspec)
if err = repo.PushContext(ctx, &git.PushOptions{
RefSpecs: []gitconfig.RefSpec{refspec},
Auth: &httpauth.BasicAuth{Username: envTOKEN, Password: envTOKEN},
Force: true,
}); err != nil {
logger.Fatal(ctx, "failed to push repo branch: %v", err)
log.Fatal(ctx, "failed to push repo branch: %v", err)
}
data := map[string]string{
@@ -564,10 +640,10 @@ func giteaPullRequest(ctx context.Context, cfg *configcli.Config, branch string,
}
if err = tplTitle.Execute(wTitle, data); err != nil {
logger.Fatal(ctx, "failed to execute template: %v", err)
log.Fatal(ctx, "failed to execute template: %v", err)
}
if err = tplBody.Execute(wBody, data); err != nil {
logger.Fatal(ctx, "failed to execute template: %v", err)
log.Fatal(ctx, "failed to execute template: %v", err)
}
body := map[string]string{
@@ -576,14 +652,14 @@ func giteaPullRequest(ctx context.Context, cfg *configcli.Config, branch string,
"head": fmt.Sprintf("pkgdash/go_modules/%s-%s", path, mod.Version),
"title": wTitle.String(),
}
logger.Info(ctx, "raw body: %#+v", body)
log.Info(ctx, "raw body: %#+v", body)
buf, err = json.Marshal(body)
if err != nil {
return err
}
logger.Info(ctx, "marshal body: %s", buf)
log.Info(ctx, "marshal body: %s", buf)
req, err := http.NewRequestWithContext(ctx, http.MethodPost, envAPIURL+"/repos/"+envREPOSITORY+"/pulls?token="+envTOKEN, bytes.NewReader(buf))
if err != nil {