unistack-org/pkgdash
4
0
Fork 1
Code Issues 2 Pull Requests 1 Packages Projects Releases Wiki Activity

#8 implement interface Source and methods for gitea without Update (#9)

Browse Source 
Прикинул метод Open и Delete, Update пока не до делал.

Co-authored-by: Gorbunov Kirill Andreevich <kgorbunov@mtsbank.ru>
Reviewed-on: #9
Co-authored-by: Кирилл Горбунов <kirya_gorbunov_2015@mail.ru>
Co-committed-by: Кирилл Горбунов <kirya_gorbunov_2015@mail.ru>
This commit is contained in:
Кирилл Горбунов
2024-03-24 20:52:32 +03:00
committed by Василий Толстов
parent c9a16829fd
commit ec5c6c591d
32 changed files with 1766 additions and 464 deletions
Download Patch File Download Diff File Expand all files Collapse all files

94
internal/cli/checkupdate.go Normal file
View File

@@ -0,0 +1,94 @@
/*
Copyright © 2024 NAME HERE <EMAIL ADDRESS>
*/
package cli
import (
"context"
"fmt"
"os"
"strings"
"git.unistack.org/unistack-org/pkgdash/internal/modules"
"github.com/spf13/cobra"
"go.unistack.org/micro/v4/logger"
"golang.org/x/mod/modfile"
"golang.org/x/mod/semver"
)
// checkupdateCmd represents the checkupdate command
var checkupdateCmd = NewCheckUpdateCommand()
var mvs = make(map[string]modules.Update)
func init() {
rootCmd.AddCommand(checkupdateCmd)
}
func NewCheckUpdateCommand() *cobra.Command {
ctx := context.Background()
cmd := &cobra.Command{
Use: "checkupdate",
Short: "CheckUpdate collects a list of dependencies with the latest updates.",
Long: `CheckUpdate collects a list of dependencies with the latest updates.`,
RunE: func(cmd *cobra.Command, args []string) error {
fmt.Println("checkupdate called")
path := "."
if len(os.Args) > 1 {
path = os.Args[1]
}
name, err := modules.FindModFile(path)
if err != nil {
panic(err)
}
buf, err := os.ReadFile(name)
if err != nil {
panic(err)
}
mfile, err := modfile.Parse(name, buf, nil)
if err != nil {
panic(err)
}
mvs = make(map[string]modules.Update)
updateOptions := modules.UpdateOptions{
Pre: cfg.UpdateOpt.Pre,
Major: cfg.UpdateOpt.Major,
UpMajor: cfg.UpdateOpt.UpMajor,
Cached: cfg.UpdateOpt.Cached,
OnUpdate: func(u modules.Update) {
var modpath string // new mod path with major
if u.Err != nil {
logger.Error(ctx, fmt.Sprintf("%s: failed: %v", u.Module.Path, u.Err))
return
}
modpath = u.Module.Path
v := semver.Major(u.Version)
p := modules.ModPrefix(modpath)
if !strings.HasPrefix(u.Module.Version, v) && v != "v1" && v != "v0" {
switch strings.HasPrefix(u.Module.Path, "gopkg.in") {
case true:
modpath = p + "." + v
case false:
modpath = p + "/" + v
}
}
mvs[modpath] = u
},
}
for _, req := range mfile.Require {
updateOptions.Modules = append(updateOptions.Modules, req.Mod)
}
modules.Updates(updateOptions)
logger.Info(ctx, fmt.Sprintf("Modules get update: /n %s", mvs))
return nil
},
}
return cmd
}

74
internal/cli/delete.go Normal file
View File

@@ -0,0 +1,74 @@
/*
Copyright © 2024 NAME HERE <EMAIL ADDRESS>
*/
package cli
import (
"context"
"fmt"
"github.com/spf13/cobra"
)
var deleteCmd = NewDeleteCommand()
type DeleteFlags struct {
all bool
mod string
}
func init() {
rootCmd.AddCommand(deleteCmd)
}
func NewDeleteCommand() *cobra.Command {
var flags DeleteFlags
ctx := context.Background()
cmd := &cobra.Command{
Use: "delete",
Short: "Delete closes the merge requests created with a dependency update.",
Long: `Delete closes the merge requests created with a dependency update.`,
RunE: func(cmd *cobra.Command, args []string) error {
fmt.Println("delete called")
var err error
if gitsource == nil {
return errSourceNil
}
if len(prList) == 0 {
return errPRNotExist
}
if flags.all {
for _, branch := range cfg.Branches {
rMap := prList[branch]
for path, _ := range rMap {
err = gitsource.RequestClose(ctx, branch, path)
if err != nil {
return err
}
}
}
}
if flags.mod != "" {
for _, branch := range cfg.Branches {
err = gitsource.RequestClose(ctx, branch, flags.mod)
if err != nil {
return err
}
}
}
return errFlagsNotExist
},
}
cmd.Flags().BoolVarP(&flags.all, "all", "a", false, "Deletes everything depending")
cmd.Flags().StringVarP(&flags.mod, "mod", "m", "", "Deletes one dependency")
return cmd
}

121
internal/cli/init.go Normal file
View File

@@ -0,0 +1,121 @@
/*
Copyright © 2024 NAME HERE <EMAIL ADDRESS>
*/
package cli
import (
"context"
"fmt"
"path/filepath"
"git.unistack.org/unistack-org/pkgdash/internal/configcli"
"git.unistack.org/unistack-org/pkgdash/internal/source"
"github.com/spf13/cobra"
yamlcodec "go.unistack.org/micro-codec-yaml/v4"
envconfig "go.unistack.org/micro-config-env/v4"
fileconfig "go.unistack.org/micro-config-file/v4"
"go.unistack.org/micro/v4/config"
"go.unistack.org/micro/v4/logger"
"go.unistack.org/micro/v4/logger/slog"
"go.unistack.org/micro/v4/options"
)
// initCmd represents the init command
var initCmd = NewInitCommand()
var gitsource = source.SourceControl(nil)
var cfg = configcli.NewConfig()
var (
DefaultPullRequestTitle = `Bump {{.Name}} from {{.VersionOld}} to {{.VersionNew}}`
DefaultPullRequestBody = `Bumps {{.Name}} from {{.VersionOld}} to {{.VersionNew}}`
)
var (
configFiles = []string{
"dependabot.yml",
"pkgdashcli.yml",
"pkgdashcli.yaml",
}
configDirs = []string{
".gitea",
".github",
".gitlab",
}
)
func init() {
rootCmd.AddCommand(initCmd)
}
func NewInitCommand() *cobra.Command {
ctx := context.Background()
logger.DefaultLogger = slog.NewLogger()
if err := logger.DefaultLogger.Init(logger.WithCallerSkipCount(3), logger.WithLevel(logger.DebugLevel)); err != nil {
logger.Error(ctx, fmt.Sprintf("logger init error: %v", err))
}
cmd := &cobra.Command{
Use: "init",
Short: "Init fills the config with data from the configuration file.",
Long: `Init fills the config with data from the configuration file.`,
RunE: func(cmd *cobra.Command, args []string) error {
fmt.Println("init called")
var err error
if err = config.Load(ctx,
[]config.Config{
config.NewConfig(
config.Struct(cfg),
),
envconfig.NewConfig(
config.Struct(cfg),
),
},
config.LoadOverride(true),
); err != nil {
logger.Fatal(ctx, fmt.Sprintf("failed to load config: %v", err))
}
for _, configDir := range configDirs {
for _, configFile := range configFiles {
logger.Info(ctx, fmt.Sprintf("path: %s", filepath.Join(configDir, configFile)))
c := fileconfig.NewConfig(
config.AllowFail(false),
config.Struct(cfg),
options.Codec(yamlcodec.NewCodec()),
fileconfig.Path(".gitea/pkgdashcli.yaml"),
)
err = c.Init(options.Context(ctx))
if err != nil {
logger.Error(ctx, fmt.Sprintf("failed to init config: %v", err))
return err
}
if err = c.Load(ctx, config.LoadOverride(true)); err != nil {
logger.Error(ctx, fmt.Sprintf("failed to load config: %v", err))
return err
}
}
}
logger.Info(ctx, fmt.Sprintf("Load config... %s", cfg.Source.Repository))
if cfg.PullRequestBody == "" {
cfg.PullRequestBody = DefaultPullRequestBody
}
if cfg.PullRequestTitle == "" {
cfg.PullRequestTitle = DefaultPullRequestTitle
}
gitsource = source.NewSourceControl(*cfg)
return nil
},
}
return cmd
}

73
internal/cli/list.go Normal file
View File

@@ -0,0 +1,73 @@
/*
Copyright © 2024 NAME HERE <EMAIL ADDRESS>
*/
package cli
import (
"context"
"fmt"
"github.com/spf13/cobra"
"go.unistack.org/micro/v4/logger"
)
// updateCmd represents the update command
var listCmd = NewListCommand()
var prList = map[string]map[string]string{}
/* [
"master":
[
"go.unistack.org/micro/v4" : "Bump go.unistack.org/micro/v4 from v4.0.0 to v4.0.1",
"go.unistack.org/micro-client-http/v4":"Bump go.unistack.org/micro-client-http/v4 from v4.0.0 to v4.0.2",
],
"v3":
[
"go.unistack.org/micro/v3" : "Bump go.unistack.org/micro/v4 from v3.0.0 to v3.0.1",
],
] */
func init() {
rootCmd.AddCommand(listCmd)
}
func NewListCommand() *cobra.Command {
ctx := context.Background()
logger.Info(ctx, "RequestList start")
cmd := &cobra.Command{
Use: "list",
Short: "Update allows you to start the process of creating a PR in the repository",
Long: `
Update allows you to start the process of creating a PR in the repository.
Use the -a flag to update all dependencies or -m flag a specific module.
`,
RunE: func(cmd *cobra.Command, args []string) error {
if gitsource == nil {
return errSourceNil
}
if cfg == nil {
return errCfgNil
}
prList = make(map[string]map[string]string)
for _, branch := range cfg.Branches {
rMap, err := gitsource.RequestList(ctx, branch)
if err != nil {
return err
}
prList[branch] = rMap
logger.Info(ctx, fmt.Sprintf("for %s:\n%s", branch, rMap))
}
return nil
},
}
return cmd
}

42
internal/cli/root.go Normal file
View File

@@ -0,0 +1,42 @@
/*
Copyright © 2024 NAME HERE <EMAIL ADDRESS>
*/
package cli
import (
"fmt"
"os"
"github.com/spf13/cobra"
)
var (
errUpdateNotExist = fmt.Errorf("mod not in list of update. Call init")
errFlagsNotExist = fmt.Errorf("empty flags")
errSourceNil = fmt.Errorf("source nil. Call init")
errCfgNil = fmt.Errorf("cfg nil. Call init")
errPRNotExist = fmt.Errorf("pr not exist. Call list")
)
// rootCmd represents the base command when called without any subcommands
var rootCmd = &cobra.Command{
Use: "pkgdashcli <command> [flags]",
Short: "Pkgdashcli application to update dependencies.",
Long: `Pkgdashcli allows you to define a version update for a dependency and start merge requests in version control systems.`,
// Uncomment the following line if your bare application
// has an action associated with it:
// Run: func(cmd *cobra.Command, args []string) { },
}
// Execute adds all child commands to the root command and sets flags appropriately.
// This is called by main.main(). It only needs to happen once to the rootCmd.
func Execute() {
err := rootCmd.Execute()
if err != nil {
os.Exit(1)
}
}
func init() {
}

95
internal/cli/update.go Normal file
View File

@@ -0,0 +1,95 @@
/*
Copyright © 2024 NAME HERE <EMAIL ADDRESS>
*/
package cli
import (
"context"
"fmt"
"github.com/spf13/cobra"
"go.unistack.org/micro/v4/logger"
)
var updateCmd = NewUpdateCommand()
type UpdateFlags struct {
all bool
mod string
}
func init() {
rootCmd.AddCommand(updateCmd)
}
func NewUpdateCommand() *cobra.Command {
var flags UpdateFlags
ctx := context.Background()
cmd := &cobra.Command{
Use: "update",
Short: "Update allows you to start the process of creating a PR in the repository",
Long: `
Update allows you to start the process of creating a PR in the repository.
Use the -a flag to update all dependencies or -m flag a specific module.
`,
RunE: func(cmd *cobra.Command, args []string) error {
var err error
if gitsource == nil {
return errSourceNil
}
if len(mvs) == 0 {
return errUpdateNotExist
}
if flags.all {
logger.Info(ctx, "update all")
for pathMod, m := range mvs {
for _, branch := range cfg.Branches {
logger.Info(ctx, fmt.Sprintf("update mod: %s", m))
if err = gitsource.RequestOpen(ctx, branch, pathMod, m); err != nil {
logger.Error(ctx, fmt.Sprintf("PR open error: %s, base branch: %s", pathMod, branch))
return err
}
logger.Info(ctx, fmt.Sprintf("Successful update mod: %s", pathMod))
}
delete(mvs, pathMod) // todo возможно чисть не стоит и нужно просто пропускать
}
return nil
}
if flags.mod != "" {
for _, branch := range cfg.Branches {
logger.Info(ctx, fmt.Sprintf("update mod: %s", flags.mod))
if _, ok := mvs[flags.mod]; !ok {
return errUpdateNotExist
}
if err = gitsource.RequestOpen(ctx, branch, flags.mod, mvs[flags.mod]); err != nil {
logger.Error(ctx, fmt.Sprintf("PR open error: %s, base branch: %s", flags.mod, branch))
return err
}
logger.Info(ctx, fmt.Sprintf("Successful update mod: %s", flags.mod))
}
delete(mvs, flags.mod)
return nil
}
return errFlagsNotExist
},
}
cmd.Flags().BoolVarP(&flags.all, "all", "a", false, "Updates everything depending")
cmd.Flags().StringVarP(&flags.mod, "mod", "m", "", "Update one dependency")
return cmd
}

30
internal/configcli/config.go Normal file
View File

@@ -0,0 +1,30 @@
package configcli
type Config struct {
PullRequestTitle string `json:"pull_request_title" yaml:"pull_request_title"`
PullRequestBody string `json:"pull_request_body" yaml:"pull_request_body"`
Branches []string `json:"branches" yaml:"branches"`
Source *Source `json:"source" yaml:"source"`
UpdateOpt *UpdateOpt `json:"update_opt" yaml:"update_opt"`
}
type Source struct {
TypeGit string `json:"type" yaml:"type"`
Token string `json:"token" yaml:"token"`
APIURL string `json:"apiurl" yaml:"apiurl"`
Repository string `json:"repository" yaml:"repository"`
Owner string `json:"owner" yaml:"owner"`
}
type UpdateOpt struct {
Pre bool `json:"pre" yaml:"pre" default:"false"`
Major bool `json:"major" yaml:"major" default:"false"`
UpMajor bool `json:"up_major" yaml:"up_major" default:"false"`
Cached bool `json:"cached" yaml:"cached" default:"true"`
}
func NewConfig() *Config {
return &Config{
Source: &Source{},
}
}

16
internal/database/database.go
View File

@@ -80,7 +80,7 @@ func ParseDSN(cfg *appconfig.DatabaseConfig) error {
case "sqlite", "sqlite3":
u.Scheme = "sqlite"
default:
return fmt.Errorf("unknown database %s", u.Scheme)
return fmt.Error("unknown database %s", u.Scheme)
}
cfg.Type = u.Scheme
@@ -95,7 +95,7 @@ func connect(ctx context.Context, cfg *appconfig.DatabaseConfig, log logger.Logg
var db *sqlx.DB
var err error
log.Infof(ctx, "connect to %s", cfg.Type)
log.Info(ctx, "connect to %s", cfg.Type)
switch cfg.Type {
case "postgres", "pgsql", "postgresql":
db, err = connectPostgres(ctx, cfg.ConnStr)
@@ -104,7 +104,7 @@ func connect(ctx context.Context, cfg *appconfig.DatabaseConfig, log logger.Logg
db, err = connectSqlite(ctx, cfg.ConnStr)
cfg.Type = "sqlite"
default:
return nil, fmt.Errorf("unknown database type %s", cfg.Type)
return nil, fmt.Error("unknown database type %s", cfg.Type)
}
if err != nil {
@@ -128,18 +128,18 @@ func Connect(ctx context.Context, cfg *appconfig.DatabaseConfig, log logger.Logg
case "":
break
case "up":
logger.Infof(ctx, "migrate up")
logger.Info(ctx, "migrate up")
err = m.Up()
case "down":
logger.Infof(ctx, "migrate down")
logger.Info(ctx, "migrate down")
err = m.Down()
case "seed":
logger.Infof(ctx, "migrate seed")
logger.Info(ctx, "migrate seed")
if err = m.Drop(); err == nil {
err = m.Up()
}
default:
logger.Infof(ctx, "migrate version")
logger.Info(ctx, "migrate version")
v, verr := strconv.ParseUint(cfg.Type, 10, 64)
if verr != nil {
return nil, err
@@ -249,5 +249,5 @@ func (l *mLog) Verbose() bool {
}
func (l *mLog) Printf(format string, v ...interface{}) {
l.l.Infof(l.ctx, format, v...)
l.l.Info(l.ctx, format, v...)
}

2
internal/handler/handler_list.go
View File

@@ -15,7 +15,7 @@ func (h *Handler) HandlerList(ctx context.Context, req *pb.HandlerListReq, rsp *
packages, err := h.store.HandlerList(ctx, req)
if err != nil {
logger.Errorf(ctx, "error db response: %v", err)
logger.Error(ctx, "error db response: %v", err)
httpsrv.SetRspCode(ctx, http.StatusInternalServerError)
return httpsrv.SetError(NewInternalError(err))
}

2
internal/handler/package_list.go
View File

@@ -15,7 +15,7 @@ func (h *Handler) PackageList(ctx context.Context, req *pb.PackageListReq, rsp *
packages, err := h.store.PackageList(ctx, req)
if err != nil {
logger.Errorf(ctx, "error db response: %v", err)
logger.Error(ctx, "error db response: %v", err)
httpsrv.SetRspCode(ctx, http.StatusInternalServerError)
return httpsrv.SetError(NewInternalError(err))
}

2
internal/handler/package_modules.go
View File

@@ -15,7 +15,7 @@ func (h *Handler) PackageModules(ctx context.Context, req *pb.PackageModulesReq,
modules, err := h.store.PackageModules(ctx, req)
if err != nil {
logger.Errorf(ctx, "error db response: %v", err)
logger.Error(ctx, "error db response: %v", err)
httpsrv.SetRspCode(ctx, http.StatusInternalServerError)
return httpsrv.SetError(NewInternalError(err))
}

12
internal/modules/modproxy.go
View File

@@ -249,7 +249,8 @@ type UpdateOptions struct {
Modules []module.Version
Pre bool
Cached bool
Major bool
Major bool // Major true compare only major
UpMajor bool // UpMajor module up with major
}
// Updates finds updates for a set of specified modules.
@@ -275,7 +276,14 @@ func Updates(opt UpdateOptions) {
ch <- Update{Module: m, Err: err}
return nil
}
v := mod.MaxVersion("", opt.Pre)
major := semver.Major(m.Version)
var v string
switch opt.UpMajor {
case true:
v = mod.MaxVersion("", opt.Pre)
case false:
v = mod.MaxVersion(major, opt.Pre)
}
if IsNewerVersion(m.Version, v, opt.Major) {
ch <- Update{Module: m, Version: v}
}

55
internal/modules/packages_test.go Normal file
View File

@@ -0,0 +1,55 @@
package modules
import "testing"
func TestModMajor(t *testing.T) {
type args struct {
modpath string
}
var tests = []struct {
name string
args args
want string
want1 bool
}{
{"Test #1",
args{
"github.com/jackc/chunkreader/v2",
},
"v2",
true,
},
{"Test #2",
args{
"github.com/jackc/chunkreader",
},
"",
true,
},
{"Test #3",
args{
"gopkg.in/yaml.v2",
},
"v2",
true,
},
{"Test #4",
args{
"github.com/jackc/chunkreader/v1",
},
"",
false,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, got1 := ModMajor(tt.args.modpath)
if got != tt.want {
t.Error("ModMajor() got = %v, want %v", got, tt.want)
}
if got1 != tt.want1 {
t.Error("ModMajor() got1 = %v, want %v", got1, tt.want1)
}
})
}
}

438
internal/source/gitea/gitea.go Normal file
View File

@@ -0,0 +1,438 @@
package gitea
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"os"
"os/exec"
"regexp"
"strings"
"text/template"
"time"
"git.unistack.org/unistack-org/pkgdash/internal/configcli"
"git.unistack.org/unistack-org/pkgdash/internal/modules"
"github.com/go-git/go-git/v5"
gitconfig "github.com/go-git/go-git/v5/config"
"github.com/go-git/go-git/v5/plumbing"
"github.com/go-git/go-git/v5/plumbing/object"
httpauth "github.com/go-git/go-git/v5/plumbing/transport/http"
"go.unistack.org/micro/v4/logger"
)
var (
ErrPRNotExist = errors.New("pull request does not exist")
)
type Gitea struct {
URL string
Token string
PRTitle string
PRBody string
Repository string
Owner string
}
func NewGitea(cfg configcli.Config) *Gitea {
return &Gitea{
URL: cfg.Source.APIURL,
Token: os.Getenv("gitea_token"),
PRTitle: cfg.PullRequestTitle,
PRBody: cfg.PullRequestBody,
Repository: cfg.Source.Repository,
Owner: cfg.Source.Owner,
}
}
type giteaPull struct {
URL string `json:"url"`
Title string `json:"title"`
Base struct {
Ref string `json:"ref"`
} `json:"base"`
Head struct {
Ref string `json:"ref"`
} `json:"head"`
ID int64 `json:"id"`
}
func (g *Gitea) RequestOpen(ctx context.Context, branch string, path string, mod modules.Update) error {
logger.Debugf(ctx, fmt.Sprintf("RequestOpen start, mod title: %s", path))
var buf []byte
var err error
// создания шаблона названия для пулл реквеста
tplTitle, err := template.New("pull_request_title").Parse(g.PRTitle)
if err != nil {
logger.Fatal(ctx, fmt.Sprintf("failed to parse template: %v", err))
}
wTitle := bytes.NewBuffer(nil)
// создания шаблона тела для пулл реквеста
tplBody, err := template.New("pull_request_body").Parse(g.PRTitle)
if err != nil {
logger.Fatal(ctx, fmt.Sprintf("failed to parse template: %v", err))
}
wBody := bytes.NewBuffer(nil)
data := map[string]string{
"Name": path,
"VersionOld": mod.Module.Version,
"VersionNew": mod.Version,
}
if err = tplTitle.Execute(wTitle, data); err != nil {
logger.Fatal(ctx, fmt.Sprintf("failed to execute template: %v", err))
}
if err = tplBody.Execute(wBody, data); err != nil {
logger.Fatal(ctx, fmt.Sprintf("failed to execute template: %v", err))
}
// открытие гит репозитория с опцией обхода репозитория для нахождения .git
repo, err := git.PlainOpenWithOptions(".", &git.PlainOpenOptions{DetectDotGit: true})
if err != nil {
logger.Fatal(ctx, fmt.Sprintf("failed to open repo: %v", err))
}
//извлекаем ссылки с объектами из удаленного объекта??
if err = repo.FetchContext(ctx, &git.FetchOptions{
Auth: &httpauth.BasicAuth{Username: g.Token, Password: g.Token},
Force: true,
}); err != nil && err != git.NoErrAlreadyUpToDate {
logger.Fatal(ctx, fmt.Sprintf("failed to fetch repo: %v", err))
} //обновляем репозиторий
var headRef *plumbing.Reference // вроде ссылка на гит
refIter, err := repo.Branches() //получение веток
if err != nil {
logger.Fatal(ctx, fmt.Sprintf("failed to get branches: %v", err))
return err
}
for {
ref, err := refIter.Next()
if err != nil {
break
}
if strings.Contains(ref.Name().String(), branch) { //todo вот тут возможно нужно переделать
headRef = ref
break
}
} //перебираем получение ветки и когда находим нужную выходим из цикла записав ветку в headRef
refIter.Close()
if headRef == nil {
logger.Fatal(ctx, "failed to get repo branch head")
return err
} // Не получили нужную ветку
logger.Info(ctx, fmt.Sprintf("repo head %s", headRef))
wtree, err := repo.Worktree() //todo вроде рабочее дерево не нужно
if err != nil {
logger.Fatal(ctx, fmt.Sprintf("failed to get worktree: %v", err))
}
pulls, err := GetPulls(ctx, g.URL, g.Owner, g.Repository, g.Token)
if err != nil && err != ErrPRNotExist {
logger.Error(ctx, fmt.Sprintf("GetPulls error: %s", err))
return err
}
for _, pull := range pulls {
logger.Debugf(ctx, fmt.Sprintf("PULL title - %s | ref - %s", pull.Title, pull.Base.Ref))
if strings.Contains(pull.Title, path) && strings.Contains(pull.Base.Ref, branch) {
logger.Info(ctx, fmt.Sprintf("PR for %s exists %s, call RequestUpdate", path, pull.URL))
return g.RequestUpdate(ctx, branch, path, mod)
} // хотим проверить есть ли пулл реквест для этой ветки, если есть то выходим
}
logger.Info(ctx, fmt.Sprintf("update %s from %s to %s", path, mod.Module.Version, mod.Version))
logger.Info(ctx, "reset worktree")
if err = wtree.Reset(&git.ResetOptions{Mode: git.HardReset}); err != nil {
logger.Error(ctx, fmt.Sprintf("failed to reset repo branch: %v", err))
} //вроде меняем ветку todo вроде можно удалить
if err = wtree.PullContext(ctx, &git.PullOptions{
Auth: &httpauth.BasicAuth{Username: g.Token, Password: g.Token},
Depth: 1,
// RemoteURL :
Force: true,
RemoteName: "origin",
}); err != nil && err != git.NoErrAlreadyUpToDate {
logger.Error(ctx, fmt.Sprintf("failed to pull repo: %v", err)) //подтягиваем изменения с удаленого репозитория
}
logger.Info(ctx, fmt.Sprintf("checkout ref %s", headRef))
if err = wtree.Checkout(&git.CheckoutOptions{
Hash: headRef.Hash(),
Branch: plumbing.NewBranchReferenceName(fmt.Sprintf("pkgdash/go_modules/%s-%s", path, mod.Version)),
Create: true,
Force: true,
}); err != nil {
logger.Error(ctx, fmt.Sprintf("failed to checkout tree: %v", err))
return err
} //создаем новую ветку
epath, err := exec.LookPath("go")
if errors.Is(err, exec.ErrDot) {
err = nil
}
if err != nil {
logger.Fatal(ctx, fmt.Sprintf("failed to find go command: %v", err))
} // ищем go файл
var cmd *exec.Cmd
var out []byte
cmd = exec.CommandContext(ctx, epath, "mod", "edit", fmt.Sprintf("-droprequire=%s", mod.Module.Path))
if out, err = cmd.CombinedOutput(); err != nil {
logger.Fatal(ctx, fmt.Sprintf("failed to run go mod edit: %s err: %v", out, err))
}
cmd = exec.CommandContext(ctx, epath, "mod", "edit", fmt.Sprintf("-require=%s@%s", path, mod.Version))
if out, err = cmd.CombinedOutput(); err != nil {
logger.Fatal(ctx, fmt.Sprintf("failed to run go mod edit: %s err: %v", out, err))
} // пытаемся выполнить команду go mod edit с новой версией модуля
cmd = exec.CommandContext(ctx, epath, "mod", "tidy")
if out, err = cmd.CombinedOutput(); err != nil {
logger.Fatal(ctx, fmt.Sprintf("failed to run go mod tidy: %s err: %v", out, err))
} // пытаемся выполнить команду go mod tidy пытаемся подтянуть новую версию модуля
logger.Info(ctx, "worktree add go.mod")
if _, err = wtree.Add("go.mod"); err != nil {
logger.Fatal(ctx, fmt.Sprintf("failed to add file: %v", err))
}
logger.Info(ctx, "worktree add go.sum")
if _, err = wtree.Add("go.sum"); err != nil {
logger.Fatal(ctx, fmt.Sprintf("failed to add file: %v", err))
}
logger.Info(ctx, "worktree commit")
_, err = wtree.Commit(wTitle.String(), &git.CommitOptions{
Parents: []plumbing.Hash{headRef.Hash()},
Author: &object.Signature{
Name: "gitea-actions",
Email: "info@unistack.org",
When: time.Now(),
},
}) // хотим за коммитить изменения
if err != nil {
logger.Fatal(ctx, fmt.Sprintf("failed to commit: %v", err))
}
refspec := gitconfig.RefSpec(fmt.Sprintf("+refs/heads/pkgdash/go_modules/%s-%s:refs/heads/pkgdash/go_modules/%s-%s", path, mod.Version, path, mod.Version)) //todo как будто нужно переделать
logger.Info(ctx, fmt.Sprintf("try to push refspec %s", refspec))
if err = repo.PushContext(ctx, &git.PushOptions{
RefSpecs: []gitconfig.RefSpec{refspec},
Auth: &httpauth.BasicAuth{Username: g.Token, Password: g.Token},
Force: true,
}); err != nil {
logger.Fatal(ctx, fmt.Sprintf("failed to push repo branch: %v", err))
} // пытаемся за пушить изменения
body := map[string]string{
"base": branch,
"body": wBody.String(),
"head": fmt.Sprintf("pkgdash/go_modules/%s-%s", path, mod.Version),
"title": wTitle.String(),
}
logger.Info(ctx, fmt.Sprintf("raw body: %#+v", body))
buf, err = json.Marshal(body)
if err != nil {
return err
}
logger.Info(ctx, fmt.Sprintf("marshal body: %s", buf))
req, err := http.NewRequestWithContext(
ctx,
http.MethodPost,
fmt.Sprintf("%s/repos/%s/%s/pulls?token=%s", g.URL, g.Owner, g.Repository, g.Token),
bytes.NewReader(buf),
)
if err != nil {
return err
}
req.Header.Add("Accept", "application/json")
req.Header.Add("Content-Type", "application/json")
rsp, err := http.DefaultClient.Do(req)
if err != nil {
return err
} //Вроде создаем новый реквест на создание пулл реквеста
if rsp.StatusCode != http.StatusCreated {
buf, _ = io.ReadAll(rsp.Body)
return fmt.Errorf("unknown error: %s", buf)
}
logger.Info(ctx, fmt.Sprintf("PR create for %s-%s", path, mod.Version))
return nil
}
func (g *Gitea) RequestClose(ctx context.Context, branch string, path string) error {
logger.Debugf(ctx, fmt.Sprintf("RequestClose start, mod title: %s", path))
pulls, err := GetPulls(ctx, g.URL, g.Owner, g.Repository, g.Token)
if err != nil {
logger.Error(ctx, fmt.Sprintf("GetPulls error: %s", err))
return err
}
prExist := false
var b string // Name of the branch to be deleted
for _, pull := range pulls {
if strings.Contains(pull.Title, path) && pull.Base.Ref == branch {
logger.Info(ctx, fmt.Sprintf("PR for %s exists: %s", path, pull.URL))
prExist = true
b = pull.Head.Ref
}
}
if !prExist {
logger.Error(ctx, fmt.Sprintf("skip %s since pr does not exist", path))
return ErrPRNotExist
}
req, err := DeleteBranch(ctx, g.URL, g.Owner, g.Repository, b, g.Token)
if err != nil {
logger.Error(ctx, fmt.Sprintf("failed to create request for delete the branch: %s, err: %s", branch, err))
return err
}
rsp, err := http.DefaultClient.Do(req)
if err != nil {
logger.Error(ctx, fmt.Sprintf("failed to do request for delete the branch: %s, err: %s, code: %s", branch, err, rsp.StatusCode))
return err
}
logger.Info(ctx, fmt.Sprintf("Delete branch for %s successful", path))
return nil
}
func (g *Gitea) RequestUpdate(ctx context.Context, branch string, path string, mod modules.Update) error {
logger.Debugf(ctx, fmt.Sprintf("RequestUpdate start, mod title: %s", path))
pulls, err := GetPulls(ctx, g.URL, g.Owner, g.Repository, g.Token)
if err != nil {
logger.Error(ctx, fmt.Sprintf("GetPulls error: %s", err))
return err
}
prExist := false
for _, pull := range pulls {
if strings.Contains(pull.Title, path) && pull.Base.Ref == branch {
logger.Info(ctx, fmt.Sprintf("don't skip %s since pr exist %s", path, pull.URL)) //todo
tVersion := getVersions(pull.Head.Ref) //Надо взять просто из названия ветки последнюю версию
if modules.IsNewerVersion(tVersion, mod.Version, false) {
reqDel, err := DeleteBranch(ctx, g.URL, g.Owner, g.Repository, pull.Head.Ref, g.Token)
if err != nil {
logger.Error(ctx, fmt.Sprintf("Error with create request for branch: %s, err: %s", branch, err))
continue
}
rsp, err := http.DefaultClient.Do(reqDel)
if err != nil {
logger.Error(ctx, fmt.Sprintf("Error with do request for branch: %s, err: %s, code: %v", branch, err, rsp.StatusCode))
continue //думаю что если не можем удалить ветку не стоит заканчивать работу, а перейти к следующей итерации
}
logger.Info(ctx, fmt.Sprintf("Old pr %s successful delete", pull.Head.Ref))
} else {
return nil
}
prExist = true
}
}
if !prExist {
logger.Error(ctx, fmt.Sprintf("skip %s since pr does not exist", path))
return ErrPRNotExist
}
logger.Info(ctx, fmt.Sprintf("update %s from %s to %s", path, mod.Module.Version, mod.Version))
return g.RequestOpen(ctx, branch, path, mod)
}
func (g *Gitea) RequestList(ctx context.Context, branch string) (map[string]string, error) {
logger.Debugf(ctx, fmt.Sprintf("RequestList for %s", branch))
gPulls, err := GetPulls(ctx, g.URL, g.Owner, g.Repository, g.Token)
if err != nil {
return nil, err
}
var path string
rMap := make(map[string]string)
for _, pull := range gPulls {
path = strings.Split(pull.Title, " ")[1] //todo Работет только для дефолтного шаблона
rMap[path] = pull.Title
}
return rMap, nil
}
func getVersions(s string) string {
re := regexp.MustCompile("[vV][0-9]+\\.[0-9]+\\.[0-9]+")
version := re.FindString(s)
return version
}
func DeleteBranch(ctx context.Context, url, owner, repo, branch, token string) (*http.Request, error) {
var buf []byte
req, err := http.NewRequestWithContext(ctx, http.MethodDelete, fmt.Sprintf("%s/repos/%s/%s/branches/%s?token=%s", url, owner, repo, branch, token), bytes.NewReader(buf))
if err != nil {
return nil, err
}
req.Header.Add("Accept", "application/json")
req.Header.Add("Content-Type", "application/json")
return req, err
}
func GetPulls(ctx context.Context, url, owner, repo, token string) ([]*giteaPull, error) {
var pulls []*giteaPull
var err error
req, err := http.NewRequestWithContext(
ctx,
http.MethodGet,
fmt.Sprintf("%s/repos/%s/%s/pulls?state=open&token=%s", url, owner, repo, token),
nil)
if err != nil {
return nil, err
} //вроде запроса к репозиторию
req.Header.Add("Accept", "application/json")
req.Header.Add("Content-Type", "application/json")
rsp, err := http.DefaultClient.Do(req) // выполнение запроса
if err != nil {
return nil, err
}
buf, _ := io.ReadAll(rsp.Body)
switch rsp.StatusCode {
case http.StatusOK:
if err = json.Unmarshal(buf, &pulls); err != nil {
logger.Error(ctx, fmt.Sprintf("failed to decode response %s err: %v", buf, err))
return nil, err
}
return pulls, nil
case http.StatusNotFound:
logger.Info(ctx, "PL is not exist for %s", repo)
return nil, ErrPRNotExist
default:
return nil, fmt.Errorf("unknown error: %s", buf)
}
}

31
internal/source/github/github.go Normal file
View File

@@ -0,0 +1,31 @@
package github
import (
"context"
"git.unistack.org/unistack-org/pkgdash/internal/configcli"
"git.unistack.org/unistack-org/pkgdash/internal/modules"
)
type Github struct {
Token string
}
func NewGithub(cfg configcli.Config) *Github {
return &Github{
Token: cfg.Source.Token,
}
}
func (g *Github) RequestOpen(ctx context.Context, branch string, path string, mod modules.Update) error {
return nil
}
func (g *Github) RequestClose(ctx context.Context, branch string, path string) error {
return nil
}
func (g *Github) RequestUpdate(ctx context.Context, branch string, path string, mod modules.Update) error {
return nil
}
func (g *Github) RequestList(ctx context.Context, branch string) (map[string]string, error) {
return nil, nil
}

31
internal/source/gitlab/gitlab.go Normal file
View File

@@ -0,0 +1,31 @@
package gitlab
import (
"context"
"git.unistack.org/unistack-org/pkgdash/internal/configcli"
"git.unistack.org/unistack-org/pkgdash/internal/modules"
)
type Gitlab struct {
Token string
}
func NewGitlab(cfg configcli.Config) *Gitlab {
return &Gitlab{
Token: cfg.Source.Token,
}
}
func (g *Gitlab) RequestOpen(ctx context.Context, branch string, path string, mod modules.Update) error {
return nil
}
func (g *Gitlab) RequestClose(ctx context.Context, branch string, path string) error {
return nil
}
func (g *Gitlab) RequestUpdate(ctx context.Context, branch string, path string, mod modules.Update) error {
return nil
}
func (g *Gitlab) RequestList(ctx context.Context, branch string) (map[string]string, error) {
return nil, nil
}

31
internal/source/gogs/gogs.go Normal file
View File

@@ -0,0 +1,31 @@
package gogs
import (
"context"
"git.unistack.org/unistack-org/pkgdash/internal/configcli"
"git.unistack.org/unistack-org/pkgdash/internal/modules"
)
type Gogs struct {
Token string
}
func NewGogs(cfg configcli.Config) *Gogs {
return &Gogs{
Token: cfg.Source.Token,
}
}
func (g *Gogs) RequestOpen(ctx context.Context, branch string, path string, mod modules.Update) error {
return nil
}
func (g *Gogs) RequestClose(ctx context.Context, branch string, path string) error {
return nil
}
func (g *Gogs) RequestUpdate(ctx context.Context, branch string, path string, mod modules.Update) error {
return nil
}
func (g *Gogs) RequestList(ctx context.Context, branch string) (map[string]string, error) {
return nil, nil
}

33
internal/source/source.go Normal file
View File

@@ -0,0 +1,33 @@
package source
import (
"context"
"git.unistack.org/unistack-org/pkgdash/internal/configcli"
"git.unistack.org/unistack-org/pkgdash/internal/modules"
"git.unistack.org/unistack-org/pkgdash/internal/source/gitea"
"git.unistack.org/unistack-org/pkgdash/internal/source/github"
"git.unistack.org/unistack-org/pkgdash/internal/source/gitlab"
"git.unistack.org/unistack-org/pkgdash/internal/source/gogs"
)
type SourceControl interface {
RequestOpen(ctx context.Context, branch string, path string, mod modules.Update) error
RequestClose(ctx context.Context, branch string, path string) error
RequestUpdate(ctx context.Context, branch string, path string, mod modules.Update) error
RequestList(ctx context.Context, branch string) (map[string]string, error)
}
func NewSourceControl(cfg configcli.Config) SourceControl {
switch cfg.Source.TypeGit {
case "github":
return github.NewGithub(cfg)
case "gitlab":
return gitlab.NewGitlab(cfg)
case "gitea":
return gitea.NewGitea(cfg)
case "gogs":
return gogs.NewGogs(cfg)
}
return nil
}

6
internal/storage/postgres_tmp/storage.go
View File

@@ -129,7 +129,7 @@ func (s *Postgres) CommentsCreate(ctx context.Context, req *pb.CommentsCreateReq
defer func() {
if err != nil {
if rollbackErr := tx.Rollback(); rollbackErr != nil {
logger.Errorf(ctx, "AddComment: unable to rollback: %v", rollbackErr)
logger.Error(ctx, "AddComment: unable to rollback: %v", rollbackErr)
}
} else {
err = tx.Commit()
@@ -159,7 +159,7 @@ func (s *Postgres) PackagesCreate(ctx context.Context, req *pb.PackagesCreateReq
defer func() {
if err != nil {
if rollbackErr := tx.Rollback(); rollbackErr != nil {
logger.Errorf(ctx, "AddPackage: unable to rollback: %v", rollbackErr)
logger.Error(ctx, "AddPackage: unable to rollback: %v", rollbackErr)
}
} else {
err = tx.Commit()
@@ -188,7 +188,7 @@ func (s *Postgres) InsertButchModules(ctx context.Context, req []models.Module)
defer func() {
if err != nil {
if rollbackErr := tx.Rollback(); rollbackErr != nil {
logger.Errorf(ctx, "AddPackage: unable to rollback: %v", rollbackErr)
logger.Error(ctx, "AddPackage: unable to rollback: %v", rollbackErr)
}
} else {
err = tx.Commit()

6
internal/storage/sqlite/storage.go
View File

@@ -64,11 +64,11 @@ func (s *Sqlite) PackageModulesCreate(ctx context.Context, pkg *models.Package,
}
func (s *Sqlite) PackageDelete(ctx context.Context, req *pb.PackageDeleteReq) error {
return fmt.Errorf("need implement")
return fmt.Error("need implement")
}
func (s *Sqlite) PackageUpdate(ctx context.Context, req *pb.PackageUpdateReq) (*models.Package, error) {
return nil, fmt.Errorf("need implement")
return nil, fmt.Error("need implement")
}
func (s *Sqlite) PackageLookup(ctx context.Context, req *pb.PackageLookupReq) (*models.Package, error) {
@@ -126,7 +126,7 @@ func (s *Sqlite) CommentCreate(ctx context.Context, req *pb.CommentCreateReq) (*
defer func() {
if err != nil {
if rollbackErr := tx.Rollback(); rollbackErr != nil {
logger.Errorf(ctx, "AddComment: unable to rollback: %v", rollbackErr)
logger.Error(ctx, "AddComment: unable to rollback: %v", rollbackErr)
}
} else {
err = tx.Commit()

20
internal/worker/worker.go
View File

@@ -42,13 +42,13 @@ func Run(ctx context.Context, store storage.Storage, td time.Duration) {
if err != sql.ErrNoRows {
continue
}
logger.Fatalf(ctx, "failed to get packages to process: %v", err)
logger.Fatal(ctx, "failed to get packages to process: %v", err)
}
wg.Add(len(packages))
for _, pkg := range packages {
go func(p *models.Package) {
if err := parseModFile(ctx, store, p); err != nil {
logger.Errorf(ctx, "failed to process package %s: %v", p.Name, err)
logger.Error(ctx, "failed to process package %s: %v", p.Name, err)
}
p.LastCheck.Time = time.Now()
wg.Done()
@@ -56,7 +56,7 @@ func Run(ctx context.Context, store storage.Storage, td time.Duration) {
}
wg.Wait()
if err = store.PackagesUpdateLastCheck(ctx, packages); err != nil {
logger.Errorf(ctx, "update packages last_check %#+v, err: %v", packages, err)
logger.Error(ctx, "update packages last_check %#+v, err: %v", packages, err)
}
case <-modTicker.C:
modules, err := store.ModulesProcess(ctx, td)
@@ -64,17 +64,17 @@ func Run(ctx context.Context, store storage.Storage, td time.Duration) {
if err != sql.ErrNoRows {
continue
}
logger.Fatalf(ctx, "failed to get modules to process: %v", err)
logger.Fatal(ctx, "failed to get modules to process: %v", err)
}
if err := processModules(ctx, store, modules); err != nil {
logger.Errorf(ctx, "failed to process modules: %v", err)
logger.Error(ctx, "failed to process modules: %v", err)
}
}
}
}
func parseModFile(ctx context.Context, store storage.Storage, pkg *models.Package) error {
logger.Infof(ctx, "process package %v", pkg)
logger.Info(ctx, "process package %v", pkg)
u, err := url.Parse(pkg.URL)
if err != nil {
@@ -107,12 +107,12 @@ func parseModFile(ctx context.Context, store storage.Storage, pkg *models.Packag
ref, err := repo.Head()
if err != nil {
return fmt.Errorf("failed to get head: %v", err)
return fmt.Error("failed to get head: %v", err)
}
commit, err := repo.CommitObject(ref.Hash())
if err != nil {
return fmt.Errorf("failed to get commit: %v", err)
return fmt.Error("failed to get commit: %v", err)
}
tree, err := commit.Tree()
@@ -156,7 +156,7 @@ func parseModFile(ctx context.Context, store storage.Storage, pkg *models.Packag
})
if err = store.PackageModulesCreate(ctx, pkg, modules); err != nil {
logger.Errorf(ctx, "failed to set create modules: %v", err)
logger.Error(ctx, "failed to set create modules: %v", err)
return err
}
@@ -182,7 +182,7 @@ func processModules(ctx context.Context, store storage.Storage, mods []*models.M
Modules: mvsu,
OnUpdate: func(u modules.Update) {
if u.Err != nil {
logger.Errorf(ctx, "%s: failed: %v", u.Module.Path, u.Err)
logger.Error(ctx, "%s: failed: %v", u.Module.Path, u.Err)
} else {
mvs[u.Module.Path].Version = u.Version
}