#19 reset HEAD
This commit is contained in:
Gorbunov Kirill Andreevich
95
internal/analyzer/coverage/coverage.go
Normal file
95
internal/analyzer/coverage/coverage.go
Normal file
@@ -0,0 +1,95 @@
|
||||
package coverage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"strings"
|
||||
|
||||
"go.unistack.org/pkgdash/internal/models"
|
||||
"golang.org/x/tools/cover"
|
||||
)
|
||||
|
||||
func Analyze(ctx context.Context, dataCoverage io.Reader, pack models.Package) (float64, error) {
|
||||
calculcate, err := calculateFiles(dataCoverage)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
mapCover := make(map[string]float64)
|
||||
{
|
||||
tree, err := GetTreeFromGit(ctx, pack.URL)
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
list, err := tree.GoFileList("")
|
||||
if err != nil {
|
||||
return 0, err
|
||||
}
|
||||
|
||||
for _, f := range list {
|
||||
mapCover[f] = 0.0
|
||||
}
|
||||
}
|
||||
|
||||
cur := len(mapCover)
|
||||
|
||||
for _, d := range calculcate.Files {
|
||||
file := strings.TrimPrefix(d.Name, pack.Name+"/")
|
||||
mapCover[file] = d.Coverage
|
||||
}
|
||||
|
||||
// check)
|
||||
if len(mapCover) != cur {
|
||||
fmt.Printf("add new keys, was: %d, has: %d", cur, len(mapCover))
|
||||
}
|
||||
|
||||
// TODO add calculate full
|
||||
|
||||
return 0, nil
|
||||
}
|
||||
|
||||
type Data struct {
|
||||
Files []*calculateFile
|
||||
Set bool
|
||||
}
|
||||
|
||||
type calculateFile struct {
|
||||
Name string
|
||||
Coverage float64
|
||||
}
|
||||
|
||||
func calculateFiles(coverSrc io.Reader) (d *Data, err error) {
|
||||
profiles, err := cover.ParseProfilesFromReader(coverSrc)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
d = new(Data)
|
||||
for _, profile := range profiles {
|
||||
fn := profile.FileName
|
||||
if profile.Mode == "set" {
|
||||
d.Set = true
|
||||
}
|
||||
|
||||
d.Files = append(d.Files, &calculateFile{
|
||||
Name: fn,
|
||||
Coverage: percentCovered(profile),
|
||||
})
|
||||
}
|
||||
return d, err
|
||||
}
|
||||
|
||||
func percentCovered(p *cover.Profile) float64 {
|
||||
var total, covered int64
|
||||
for _, b := range p.Blocks {
|
||||
total += int64(b.NumStmt)
|
||||
if b.Count > 0 {
|
||||
covered += int64(b.NumStmt)
|
||||
}
|
||||
}
|
||||
if total == 0 {
|
||||
return 0
|
||||
}
|
||||
return float64(covered) / float64(total) * 100
|
||||
}
|
||||
37
internal/analyzer/coverage/coverage_test.go
Normal file
37
internal/analyzer/coverage/coverage_test.go
Normal file
@@ -0,0 +1,37 @@
|
||||
package coverage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/stretchr/testify/assert"
|
||||
"go.unistack.org/pkgdash/internal/models"
|
||||
)
|
||||
|
||||
func Test_Calculate(t *testing.T) {
|
||||
file, err := os.Open("cover_test.out")
|
||||
assert.Nil(t, err)
|
||||
defer func() {
|
||||
assert.Nil(t, file.Close())
|
||||
}()
|
||||
|
||||
dataFiles, err := calculateFiles(file)
|
||||
assert.Nil(t, err)
|
||||
assert.NotNil(t, dataFiles)
|
||||
}
|
||||
|
||||
func Test_Analyze(t *testing.T) {
|
||||
file, err := os.Open("cover_test.out")
|
||||
assert.Nil(t, err)
|
||||
defer func() {
|
||||
assert.Nil(t, file.Close())
|
||||
}()
|
||||
|
||||
analyze, err := Analyze(context.Background(), file, models.Package{
|
||||
Name: "go.unistack.org/micro/v3",
|
||||
URL: "https://git.unistack.org/unistack-org/micro.git",
|
||||
})
|
||||
assert.Nil(t, err)
|
||||
assert.Equal(t, analyze, 0.0)
|
||||
}
|
||||
70
internal/analyzer/coverage/git.go
Normal file
70
internal/analyzer/coverage/git.go
Normal file
@@ -0,0 +1,70 @@
|
||||
package coverage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"github.com/go-git/go-git/v5"
|
||||
"github.com/go-git/go-git/v5/plumbing/filemode"
|
||||
"github.com/go-git/go-git/v5/plumbing/object"
|
||||
"github.com/go-git/go-git/v5/storage/memory"
|
||||
"os"
|
||||
"regexp"
|
||||
"strings"
|
||||
)
|
||||
|
||||
var (
|
||||
fileNil = errors.New("file pointer is nil")
|
||||
)
|
||||
|
||||
func GetTreeFromGit(ctx context.Context, url string) (*Tree, error) {
|
||||
cloneOpts := &git.CloneOptions{
|
||||
URL: url,
|
||||
Progress: os.Stdout,
|
||||
}
|
||||
|
||||
repo, err := git.CloneContext(ctx, memory.NewStorage(), nil, cloneOpts)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
ref, err := repo.Head()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get head: %v", err)
|
||||
}
|
||||
|
||||
commit, err := repo.CommitObject(ref.Hash())
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("failed to get commit: %v", err)
|
||||
}
|
||||
|
||||
tree, err := commit.Tree()
|
||||
|
||||
return &Tree{tree}, err
|
||||
}
|
||||
|
||||
type Tree struct {
|
||||
*object.Tree
|
||||
}
|
||||
|
||||
func (t Tree) GoFileList(pattern string) ([]string, error) {
|
||||
matcher, err := regexp.Compile(pattern)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var list []string
|
||||
err = t.Files().ForEach(func(file *object.File) error {
|
||||
if file == nil {
|
||||
return fileNil
|
||||
}
|
||||
if file.Mode == filemode.Regular && strings.HasSuffix(file.Name, ".go") && !strings.HasSuffix(file.Name, "_test.go") && matcher.MatchString(file.Name) {
|
||||
list = append(list, file.Name)
|
||||
}
|
||||
return nil
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return list, err
|
||||
}
|
||||
70
internal/config/config.go
Normal file
70
internal/config/config.go
Normal file
@@ -0,0 +1,70 @@
|
||||
package config
|
||||
|
||||
import (
|
||||
"time"
|
||||
|
||||
mtime "go.unistack.org/micro/v3/util/time"
|
||||
)
|
||||
|
||||
type AppConfig struct {
|
||||
CheckInterval mtime.Duration `json:"check_interval" yaml:"check_interval" default:"1d"`
|
||||
}
|
||||
|
||||
type ServerConfig struct {
|
||||
Name string `json:"name" yaml:"name"`
|
||||
Version string `json:"-" yaml:"-"`
|
||||
Addr string `json:"addr" yaml:"addr" default:":9090"`
|
||||
Crt string `json:"crt" yaml:"crt"`
|
||||
Key string `json:"key" yaml:"key"`
|
||||
ID string `json:"-" yaml:"-" default:"micro:generate uuid"`
|
||||
LoggerLevel string `json:"logger_level" yaml:"logger_level"`
|
||||
}
|
||||
|
||||
type TracerConfig struct {
|
||||
Metadata map[string]string `json:"metadata" yaml:"metadata"`
|
||||
AgentHost string `env:"JAEGER_AGENT_HOST" json:"host" yaml:"host" default:"127.0.0.1"`
|
||||
AgentPort string `env:"JAEGER_AGENT_PORT" json:"port" yaml:"port" default:"6831"`
|
||||
Collector string `env:"JAEGER_ENDPOINT,TRACER_ENDPOINT" json:"endpoint" yaml:"endpoint"`
|
||||
}
|
||||
|
||||
type VaultConfig struct {
|
||||
Addr string `env:"VAULT_ADDR" json:"addr" yaml:"addr" default:"http://127.0.0.1:8200"`
|
||||
Token string `env:"VAULT_TOKEN" json:"-" yaml:"-"`
|
||||
Path string `env:"VAULT_PATH" json:"-" yaml:"-" default:"pkgdash/data/pkgdash"`
|
||||
}
|
||||
|
||||
type MeterConfig struct {
|
||||
Addr string `json:"addr" yaml:"addr" default:"0.0.0.0:8080"`
|
||||
Path string `json:"path" yaml:"path" default:"/metrics"`
|
||||
}
|
||||
|
||||
type DatabaseConfig struct {
|
||||
DSN string `json:"dsn" yaml:"dsn"`
|
||||
Type string `json:"-" yaml:"-"`
|
||||
Migrate string `json:"-" yaml:"-"`
|
||||
ConnStr string `json:"-" yaml:"-"`
|
||||
MaxOpenConns int `json:"-" yaml:"-"`
|
||||
MaxIdleConns int `json:"-" yaml:"-"`
|
||||
ConnMaxLifetime time.Duration `json:"-" yaml:"-"`
|
||||
ConnMaxIdleTime time.Duration `json:"-" yaml:"-"`
|
||||
MigrateForce bool `json:"-" yaml:"-"`
|
||||
}
|
||||
|
||||
type Config struct {
|
||||
App *AppConfig `json:"app" yaml:"app"`
|
||||
Database *DatabaseConfig `json:"database" yaml:"database"`
|
||||
Server *ServerConfig `json:"server" yaml:"server"`
|
||||
Meter *MeterConfig `json:"meter" yaml:"meter"`
|
||||
Vault *VaultConfig `json:"-" yaml:"-"`
|
||||
Tracer *TracerConfig `json:"tracer" yaml:"tracer"`
|
||||
}
|
||||
|
||||
func NewConfig(name, version string) *Config {
|
||||
return &Config{
|
||||
App: &AppConfig{},
|
||||
Server: &ServerConfig{Name: name, Version: version},
|
||||
Tracer: &TracerConfig{},
|
||||
Meter: &MeterConfig{},
|
||||
Vault: &VaultConfig{},
|
||||
}
|
||||
}
|
||||
37
internal/configcli/config.go
Normal file
37
internal/configcli/config.go
Normal file
@@ -0,0 +1,37 @@
|
||||
package configcli
|
||||
|
||||
type Config struct {
|
||||
PullRequestTitle string `json:"pull_request_title" yaml:"pull_request_title"`
|
||||
PullRequestBody string `json:"pull_request_body" yaml:"pull_request_body"`
|
||||
Branches []string `json:"branches" yaml:"branches"`
|
||||
Source *Source `json:"source" yaml:"source"`
|
||||
UpdateOpt *UpdateOpt `json:"update_opt" yaml:"update_opt"`
|
||||
}
|
||||
|
||||
type Source struct {
|
||||
TypeGit string `json:"type" yaml:"type" env:"GIT_TYPE"`
|
||||
Username string `json:"username" yaml:"username" env:"GIT_USERNAME"`
|
||||
Password string `json:"password" yaml:"password" env:"GIT_PASSWORD,GIT_TOKEN"`
|
||||
APIURL string `json:"apiurl" yaml:"apiurl" env:"GIT_API"`
|
||||
Repository string `json:"repository" yaml:"repository" env:"GIT_REPO"`
|
||||
Owner string `json:"owner" yaml:"owner" env:"GIT_OWNER"`
|
||||
}
|
||||
|
||||
type UpdateOpt struct {
|
||||
Pre bool `json:"pre" yaml:"pre" default:"false"`
|
||||
Major bool `json:"major" yaml:"major" default:"false"`
|
||||
UpMajor bool `json:"up_major" yaml:"up_major" default:"false"`
|
||||
Cached bool `json:"cached" yaml:"cached" default:"true"`
|
||||
}
|
||||
|
||||
type Cli struct {
|
||||
Command string `flag:"name=command,desc='choice command(update, close, checkupdaue, list)',default=''"`
|
||||
Path string `flag:"name=path,desc='title of mod',default=''"`
|
||||
}
|
||||
|
||||
func NewConfig() *Config {
|
||||
return &Config{
|
||||
Source: &Source{},
|
||||
UpdateOpt: &UpdateOpt{},
|
||||
}
|
||||
}
|
||||
253
internal/database/database.go
Normal file
253
internal/database/database.go
Normal file
@@ -0,0 +1,253 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"net/url"
|
||||
"strconv"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/golang-migrate/migrate/v4"
|
||||
"github.com/golang-migrate/migrate/v4/database"
|
||||
mpgx "github.com/golang-migrate/migrate/v4/database/pgx"
|
||||
msqlite "github.com/golang-migrate/migrate/v4/database/sqlite"
|
||||
"github.com/golang-migrate/migrate/v4/source/iofs"
|
||||
"github.com/jackc/pgx/v5"
|
||||
"github.com/jackc/pgx/v5/stdlib"
|
||||
"github.com/jmoiron/sqlx"
|
||||
"go.unistack.org/micro/v3/logger"
|
||||
appconfig "go.unistack.org/pkgdash/internal/config"
|
||||
_ "modernc.org/sqlite"
|
||||
)
|
||||
|
||||
func ParseDSN(cfg *appconfig.DatabaseConfig) error {
|
||||
var err error
|
||||
|
||||
u, err := url.Parse(cfg.DSN)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
values := u.Query()
|
||||
var value string
|
||||
|
||||
if value = values.Get("conn_max"); value != "" {
|
||||
values.Del("conn_max")
|
||||
maxOpenConns, err := strconv.Atoi(value)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
cfg.MaxOpenConns = maxOpenConns
|
||||
cfg.MaxIdleConns = maxOpenConns / 2
|
||||
}
|
||||
|
||||
if value = values.Get("conn_maxidle"); value != "" {
|
||||
values.Del("conn_maxidle")
|
||||
maxIdleConns, err := strconv.Atoi(value)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
cfg.MaxIdleConns = maxIdleConns
|
||||
}
|
||||
|
||||
if value = values.Get("conn_lifetime"); value != "" {
|
||||
values.Del("conn_lifetime")
|
||||
connMaxLifetime, err := time.ParseDuration(value)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
cfg.ConnMaxLifetime = connMaxLifetime
|
||||
}
|
||||
|
||||
if value = values.Get("conn_maxidletime"); value != "" {
|
||||
values.Del("conn_maxidletime")
|
||||
connMaxIdleTime, err := time.ParseDuration(value)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
cfg.ConnMaxIdleTime = connMaxIdleTime
|
||||
}
|
||||
|
||||
if mtype := values.Get("migrate"); mtype != "" {
|
||||
values.Del("migrate")
|
||||
cfg.Migrate = mtype
|
||||
}
|
||||
|
||||
switch u.Scheme {
|
||||
case "postgres", "pgsql", "postgresql":
|
||||
u.Scheme = "postgres"
|
||||
case "sqlite", "sqlite3":
|
||||
u.Scheme = "sqlite"
|
||||
default:
|
||||
return fmt.Errorf("unknown database %s", u.Scheme)
|
||||
}
|
||||
|
||||
cfg.Type = u.Scheme
|
||||
u.RawQuery = values.Encode()
|
||||
|
||||
cfg.ConnStr = u.String()
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func connect(ctx context.Context, cfg *appconfig.DatabaseConfig, log logger.Logger) (*sqlx.DB, error) {
|
||||
var db *sqlx.DB
|
||||
var err error
|
||||
|
||||
log.Info(ctx, "connect to %s", cfg.Type)
|
||||
switch cfg.Type {
|
||||
case "postgres", "pgsql", "postgresql":
|
||||
db, err = connectPostgres(ctx, cfg.ConnStr)
|
||||
cfg.Type = "postgres"
|
||||
case "sqlite", "sqlite3":
|
||||
db, err = connectSqlite(ctx, cfg.ConnStr)
|
||||
cfg.Type = "sqlite"
|
||||
default:
|
||||
return nil, fmt.Errorf("unknown database type %s", cfg.Type)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return db, nil
|
||||
}
|
||||
|
||||
func Connect(ctx context.Context, cfg *appconfig.DatabaseConfig, log logger.Logger) (*sqlx.DB, error) {
|
||||
db, err := connect(ctx, cfg, log)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
m, err := migratePrepare(ctx, db, log, cfg.Type)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
switch cfg.Migrate {
|
||||
case "":
|
||||
break
|
||||
case "up":
|
||||
log.Info(ctx, "migrate up")
|
||||
err = m.Up()
|
||||
case "down":
|
||||
log.Info(ctx, "migrate down")
|
||||
err = m.Down()
|
||||
case "seed":
|
||||
log.Info(ctx, "migrate seed")
|
||||
if err = m.Drop(); err == nil {
|
||||
err = m.Up()
|
||||
}
|
||||
default:
|
||||
log.Info(ctx, "migrate version")
|
||||
v, verr := strconv.ParseUint(cfg.Type, 10, 64)
|
||||
if verr != nil {
|
||||
return nil, err
|
||||
}
|
||||
err = m.Migrate(uint(v))
|
||||
}
|
||||
|
||||
if err == nil || err == migrate.ErrNoChange {
|
||||
srcerr, dberr := m.Close()
|
||||
if srcerr != nil {
|
||||
err = srcerr
|
||||
} else if dberr != nil {
|
||||
err = dberr
|
||||
} else {
|
||||
err = nil
|
||||
}
|
||||
}
|
||||
|
||||
if err == nil {
|
||||
db, err = connect(ctx, cfg, log)
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
db.SetConnMaxIdleTime(cfg.ConnMaxIdleTime)
|
||||
db.SetConnMaxLifetime(cfg.ConnMaxLifetime)
|
||||
db.SetMaxIdleConns(cfg.MaxIdleConns)
|
||||
db.SetMaxOpenConns(cfg.MaxOpenConns)
|
||||
|
||||
return db, nil
|
||||
}
|
||||
|
||||
func connectSqlite(ctx context.Context, connstr string) (*sqlx.DB, error) {
|
||||
if !strings.Contains(connstr, ":memory:") {
|
||||
return sqlx.ConnectContext(ctx, "sqlite", "file:"+connstr[9:])
|
||||
}
|
||||
return sqlx.ConnectContext(ctx, "sqlite", connstr[9:])
|
||||
}
|
||||
|
||||
func connectPostgres(ctx context.Context, connstr string) (*sqlx.DB, error) {
|
||||
// parse connection string
|
||||
dbConf, err := pgx.ParseConfig(connstr)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// needed for pgbouncer
|
||||
dbConf.RuntimeParams = map[string]string{
|
||||
"standard_conforming_strings": "on",
|
||||
"application_name": "authn",
|
||||
}
|
||||
|
||||
// may be needed for pbbouncer, needs to check
|
||||
// dbConf.PreferSimpleProtocol = true
|
||||
// register pgx conn
|
||||
connStr := stdlib.RegisterConnConfig(dbConf)
|
||||
|
||||
db, err := sqlx.ConnectContext(ctx, "pgx", connStr)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return db, nil
|
||||
}
|
||||
|
||||
func migratePrepare(ctx context.Context, db *sqlx.DB, log logger.Logger, dbtype string) (*migrate.Migrate, error) {
|
||||
var driver database.Driver
|
||||
var err error
|
||||
|
||||
switch dbtype {
|
||||
case "postgres":
|
||||
driver, err = mpgx.WithInstance(db.DB, &mpgx.Config{
|
||||
DatabaseName: "pkgdash",
|
||||
MigrationsTable: "schema_migrations",
|
||||
})
|
||||
case "sqlite":
|
||||
driver, err = msqlite.WithInstance(db.DB, &msqlite.Config{
|
||||
DatabaseName: "pkgdash",
|
||||
MigrationsTable: "schema_migrations",
|
||||
})
|
||||
}
|
||||
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
source, err := iofs.New(assets, "migrations/"+dbtype)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
m, err := migrate.NewWithInstance("fs", source, "apigw", driver)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
m.Log = &mLog{ctx: ctx, l: log}
|
||||
return m, nil
|
||||
}
|
||||
|
||||
type mLog struct {
|
||||
ctx context.Context
|
||||
l logger.Logger
|
||||
}
|
||||
|
||||
func (l *mLog) Verbose() bool {
|
||||
return l.l.V(logger.DebugLevel)
|
||||
}
|
||||
|
||||
func (l *mLog) Printf(format string, v ...interface{}) {
|
||||
l.l.Info(l.ctx, format, v...)
|
||||
}
|
||||
8
internal/database/embed.go
Normal file
8
internal/database/embed.go
Normal file
@@ -0,0 +1,8 @@
|
||||
package database
|
||||
|
||||
import (
|
||||
"embed"
|
||||
)
|
||||
|
||||
//go:embed migrations
|
||||
var assets embed.FS
|
||||
@@ -0,0 +1 @@
|
||||
drop table if exists dashboard, package, module, issue, comment;
|
||||
@@ -0,0 +1,39 @@
|
||||
create table if not exists dashboard (
|
||||
id serial not null unique primary key ,
|
||||
"uuid" uuid not null unique default gen_random_uuid() ,
|
||||
package integer[] default '{}'::integer[]
|
||||
);
|
||||
|
||||
create table if not exists comment (
|
||||
id serial not null unique primary key ,
|
||||
"text" text ,
|
||||
package integer not null,
|
||||
created timestamp not null default current_timestamp ,
|
||||
updated timestamp default current_timestamp
|
||||
);
|
||||
|
||||
create table if not exists module (
|
||||
id serial not null unique primary key ,
|
||||
name varchar not null ,
|
||||
version varchar not null
|
||||
);
|
||||
|
||||
create table if not exists issue (
|
||||
id serial not null unique primary key ,
|
||||
--package integer references package(id) ,
|
||||
modules integer[] default '{}'::integer[],
|
||||
status integer default 0 ,
|
||||
"desc" varchar
|
||||
);
|
||||
|
||||
create table if not exists package (
|
||||
id serial not null unique primary key ,
|
||||
name varchar not null ,
|
||||
url varchar ,
|
||||
modules integer[] default '{}'::integer[],
|
||||
issues integer[] default '{}'::integer[],
|
||||
comments integer[] default '{}'::integer[]
|
||||
);
|
||||
|
||||
create unique index module_info on module(name, version);
|
||||
|
||||
@@ -0,0 +1,5 @@
|
||||
drop table if exists packages;
|
||||
drop table if exists modules;
|
||||
drop table if exists issues;
|
||||
drop table if exists comments;
|
||||
drop table if exists handlers;
|
||||
@@ -0,0 +1,57 @@
|
||||
create table if not exists comments (
|
||||
id integer primary key autoincrement not null,
|
||||
comment text,
|
||||
package integer not null,
|
||||
created timestamp not null default current_timestamp,
|
||||
updated timestamp not null default current_timestamp
|
||||
);
|
||||
|
||||
|
||||
create table if not exists issues (
|
||||
id integer primary key autoincrement not null,
|
||||
status integer default 0,
|
||||
comment varchar,
|
||||
created timestamp not null default current_timestamp,
|
||||
updated timestamp not null default current_timestamp
|
||||
);
|
||||
|
||||
create table if not exists handlers (
|
||||
id integer primary key autoincrement not null,
|
||||
package integer not null,
|
||||
name varchar,
|
||||
coverage number default 0
|
||||
);
|
||||
|
||||
create table if not exists packages (
|
||||
id integer primary key autoincrement not null,
|
||||
name varchar not null,
|
||||
url varchar not null,
|
||||
description varchar,
|
||||
modules integer default 0,
|
||||
issues integer default 0,
|
||||
comments integer default 0,
|
||||
coverage number default 0,
|
||||
created timestamp not null default current_timestamp,
|
||||
updated timestamp not null default current_timestamp,
|
||||
status integer default 1,
|
||||
last_check timestamp
|
||||
);
|
||||
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS unique_idx_url on packages (url);
|
||||
|
||||
create table if not exists modules (
|
||||
id integer primary key autoincrement not null,
|
||||
name varchar not null,
|
||||
version varchar not null,
|
||||
last_check timestamp not null default current_timestamp
|
||||
);
|
||||
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS unique_idx_name_version on modules (name,version);
|
||||
|
||||
create table if not exists packages_modules (
|
||||
id integer primary key autoincrement not null,
|
||||
package integer,
|
||||
module integer not null
|
||||
);
|
||||
|
||||
CREATE UNIQUE INDEX IF NOT EXISTS unique_idx_package_module on packages_modules (package,module);
|
||||
39
internal/handler/comment_create.go
Normal file
39
internal/handler/comment_create.go
Normal file
@@ -0,0 +1,39 @@
|
||||
package handler
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"net/http"
|
||||
|
||||
httpsrv "go.unistack.org/micro-server-http/v3"
|
||||
"go.unistack.org/pkgdash/internal/models"
|
||||
pb "go.unistack.org/pkgdash/proto"
|
||||
)
|
||||
|
||||
func (h *Handler) CommentCreate(ctx context.Context, req *pb.CommentCreateReq, rsp *pb.CommentCreateRsp) error {
|
||||
h.logger.Debug(ctx, "Start AddComment")
|
||||
|
||||
err := req.Validate()
|
||||
if err != nil {
|
||||
h.logger.Error(ctx, "validation error", err)
|
||||
httpsrv.SetRspCode(ctx, http.StatusBadRequest)
|
||||
return httpsrv.SetError(NewValidationError(err))
|
||||
}
|
||||
|
||||
var com *models.Comment
|
||||
if com, err = h.store.CommentCreate(ctx, req); err != nil {
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
httpsrv.SetRspCode(ctx, http.StatusNotFound)
|
||||
return httpsrv.SetError(NewNotFoundError(err))
|
||||
}
|
||||
h.logger.Error(ctx, "comment create error", err)
|
||||
httpsrv.SetRspCode(ctx, http.StatusInternalServerError)
|
||||
return httpsrv.SetError(NewInternalError(err))
|
||||
}
|
||||
|
||||
rsp.Comment = models.NewComment(com)
|
||||
|
||||
h.logger.Debug(ctx, "Success finish addComment")
|
||||
return nil
|
||||
}
|
||||
36
internal/handler/comment_delete.go
Normal file
36
internal/handler/comment_delete.go
Normal file
@@ -0,0 +1,36 @@
|
||||
package handler
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"errors"
|
||||
"net/http"
|
||||
|
||||
httpsrv "go.unistack.org/micro-server-http/v3"
|
||||
pb "go.unistack.org/pkgdash/proto"
|
||||
)
|
||||
|
||||
func (h *Handler) CommentDelete(ctx context.Context, req *pb.CommentDeleteReq, rsp *pb.CommentDeleteRsp) error {
|
||||
h.logger.Debug(ctx, "Start AddComment")
|
||||
|
||||
err := req.Validate()
|
||||
if err != nil {
|
||||
h.logger.Error(ctx, "validate error", err)
|
||||
httpsrv.SetRspCode(ctx, http.StatusBadRequest)
|
||||
return httpsrv.SetError(NewValidationError(err))
|
||||
}
|
||||
|
||||
if err = h.store.CommentDelete(ctx, req); err != nil {
|
||||
|
||||
if errors.Is(err, sql.ErrNoRows) {
|
||||
httpsrv.SetRspCode(ctx, http.StatusNotFound)
|
||||
return httpsrv.SetError(NewNotFoundError(err))
|
||||
}
|
||||
h.logger.Error(ctx, "comment delete error", err)
|
||||
httpsrv.SetRspCode(ctx, http.StatusInternalServerError)
|
||||
return httpsrv.SetError(NewInternalError(err))
|
||||
}
|
||||
|
||||
h.logger.Debug(ctx, "Success finish addComment")
|
||||
return nil
|
||||
}
|
||||
35
internal/handler/comment_list.go
Normal file
35
internal/handler/comment_list.go
Normal file
@@ -0,0 +1,35 @@
|
||||
package handler
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
|
||||
httpsrv "go.unistack.org/micro-server-http/v3"
|
||||
"go.unistack.org/pkgdash/internal/models"
|
||||
pb "go.unistack.org/pkgdash/proto"
|
||||
)
|
||||
|
||||
func (h *Handler) CommentList(ctx context.Context, req *pb.CommentListReq, rsp *pb.CommentListRsp) error {
|
||||
h.logger.Debug(ctx, "Start GetModule")
|
||||
|
||||
err := req.Validate()
|
||||
if err != nil {
|
||||
h.logger.Error(ctx, "validate error", err)
|
||||
httpsrv.SetRspCode(ctx, http.StatusBadRequest)
|
||||
return httpsrv.SetError(NewValidationError(err))
|
||||
}
|
||||
|
||||
comments, err := h.store.CommentList(ctx, req)
|
||||
if err != nil {
|
||||
h.logger.Error(ctx, "comment list error", err)
|
||||
httpsrv.SetRspCode(ctx, http.StatusInternalServerError)
|
||||
return httpsrv.SetError(NewInternalError(err))
|
||||
}
|
||||
|
||||
for _, com := range comments {
|
||||
rsp.Comments = append(rsp.Comments, models.NewComment(com))
|
||||
}
|
||||
|
||||
h.logger.Debug(ctx, "Success finish getModule")
|
||||
return nil
|
||||
}
|
||||
11
internal/handler/comment_lookup.go
Normal file
11
internal/handler/comment_lookup.go
Normal file
@@ -0,0 +1,11 @@
|
||||
package handler
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
pb "go.unistack.org/pkgdash/proto"
|
||||
)
|
||||
|
||||
func (h *Handler) CommentLookup(ctx context.Context, req *pb.CommentLookupReq, rsp *pb.CommentLookupRsp) error {
|
||||
return nil
|
||||
}
|
||||
56
internal/handler/handler.go
Normal file
56
internal/handler/handler.go
Normal file
@@ -0,0 +1,56 @@
|
||||
package handler
|
||||
|
||||
import (
|
||||
"net/http"
|
||||
"strconv"
|
||||
|
||||
"github.com/google/uuid"
|
||||
jsonpbcodec "go.unistack.org/micro-codec-jsonpb/v3"
|
||||
"go.unistack.org/micro/v3/codec"
|
||||
"go.unistack.org/micro/v3/logger"
|
||||
"go.unistack.org/pkgdash/internal/storage"
|
||||
pb "go.unistack.org/pkgdash/proto"
|
||||
)
|
||||
|
||||
type Handler struct {
|
||||
logger logger.Logger
|
||||
store storage.Storage
|
||||
codec codec.Codec
|
||||
}
|
||||
|
||||
func NewNotFoundError(err error) *pb.ErrorRsp {
|
||||
return &pb.ErrorRsp{
|
||||
Code: strconv.Itoa(http.StatusBadRequest),
|
||||
Title: "NotFound",
|
||||
Uuid: uuid.New().String(),
|
||||
Details: err.Error(),
|
||||
}
|
||||
}
|
||||
|
||||
func NewInternalError(err error) *pb.ErrorRsp {
|
||||
return &pb.ErrorRsp{
|
||||
Code: strconv.Itoa(http.StatusInternalServerError),
|
||||
Title: "InternalServerError",
|
||||
Uuid: uuid.New().String(),
|
||||
Details: err.Error(),
|
||||
}
|
||||
}
|
||||
|
||||
func NewValidationError(err error) *pb.ErrorRsp {
|
||||
return &pb.ErrorRsp{
|
||||
Code: strconv.Itoa(http.StatusBadRequest),
|
||||
Title: "BadRequest",
|
||||
Uuid: uuid.New().String(),
|
||||
Details: err.Error(),
|
||||
}
|
||||
}
|
||||
|
||||
func NewHandler(log logger.Logger, store storage.Storage) (*Handler, error) {
|
||||
h := &Handler{
|
||||
logger: log,
|
||||
codec: jsonpbcodec.NewCodec(),
|
||||
store: store,
|
||||
}
|
||||
|
||||
return h, nil
|
||||
}
|
||||
27
internal/handler/handler_list.go
Normal file
27
internal/handler/handler_list.go
Normal file
@@ -0,0 +1,27 @@
|
||||
package handler
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
|
||||
httpsrv "go.unistack.org/micro-server-http/v3"
|
||||
"go.unistack.org/pkgdash/internal/models"
|
||||
pb "go.unistack.org/pkgdash/proto"
|
||||
)
|
||||
|
||||
func (h *Handler) HandlerList(ctx context.Context, req *pb.HandlerListReq, rsp *pb.HandlerListRsp) error {
|
||||
h.logger.Debug(ctx, "HandlerList handler start")
|
||||
|
||||
packages, err := h.store.HandlerList(ctx, req)
|
||||
if err != nil {
|
||||
h.logger.Error(ctx, "error db response: %v", err)
|
||||
httpsrv.SetRspCode(ctx, http.StatusInternalServerError)
|
||||
return httpsrv.SetError(NewInternalError(err))
|
||||
}
|
||||
|
||||
for _, hdlr := range packages {
|
||||
rsp.Handlers = append(rsp.Handlers, models.NewHandler(hdlr))
|
||||
}
|
||||
h.logger.Debug(ctx, "HandlerList handler stop")
|
||||
return nil
|
||||
}
|
||||
34
internal/handler/modules_list.go
Normal file
34
internal/handler/modules_list.go
Normal file
@@ -0,0 +1,34 @@
|
||||
package handler
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
|
||||
httpsrv "go.unistack.org/micro-server-http/v3"
|
||||
"go.unistack.org/pkgdash/internal/models"
|
||||
pb "go.unistack.org/pkgdash/proto"
|
||||
)
|
||||
|
||||
func (h *Handler) ModuleList(ctx context.Context, req *pb.ModuleListReq, rsp *pb.ModuleListRsp) error {
|
||||
h.logger.Debug(ctx, "Start GetModule")
|
||||
|
||||
err := req.Validate()
|
||||
if err != nil {
|
||||
h.logger.Error(ctx, "validate error", err)
|
||||
httpsrv.SetRspCode(ctx, http.StatusBadRequest)
|
||||
return httpsrv.SetError(NewValidationError(err))
|
||||
}
|
||||
|
||||
modules, err := h.store.ModuleList(ctx, req)
|
||||
if err != nil {
|
||||
h.logger.Error(ctx, "module list error", err)
|
||||
httpsrv.SetRspCode(ctx, http.StatusInternalServerError)
|
||||
return httpsrv.SetError(NewInternalError(err))
|
||||
}
|
||||
|
||||
for _, mod := range modules {
|
||||
rsp.Modules = append(rsp.Modules, models.NewModule(mod))
|
||||
}
|
||||
h.logger.Debug(ctx, "Success finish getModule")
|
||||
return nil
|
||||
}
|
||||
32
internal/handler/package_create.go
Normal file
32
internal/handler/package_create.go
Normal file
@@ -0,0 +1,32 @@
|
||||
package handler
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
|
||||
httpsrv "go.unistack.org/micro-server-http/v3"
|
||||
"go.unistack.org/pkgdash/internal/models"
|
||||
pb "go.unistack.org/pkgdash/proto"
|
||||
)
|
||||
|
||||
func (h *Handler) PackageCreate(ctx context.Context, req *pb.PackageCreateReq, rsp *pb.PackageCreateRsp) error {
|
||||
h.logger.Debug(ctx, "PackagesCreate handler start")
|
||||
|
||||
if err := req.Validate(); err != nil {
|
||||
h.logger.Error(ctx, "validate error", err)
|
||||
httpsrv.SetRspCode(ctx, http.StatusBadRequest)
|
||||
return httpsrv.SetError(NewValidationError(err))
|
||||
}
|
||||
|
||||
pkg, err := h.store.PackageCreate(ctx, req)
|
||||
if err != nil {
|
||||
h.logger.Error(ctx, "package create error", err)
|
||||
httpsrv.SetRspCode(ctx, http.StatusBadRequest)
|
||||
return httpsrv.SetError(NewValidationError(err))
|
||||
}
|
||||
|
||||
rsp.Package = models.NewPackage(pkg)
|
||||
|
||||
h.logger.Debug(ctx, "PackagesCreate handler stop")
|
||||
return nil
|
||||
}
|
||||
28
internal/handler/package_delete.go
Normal file
28
internal/handler/package_delete.go
Normal file
@@ -0,0 +1,28 @@
|
||||
package handler
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
|
||||
httpsrv "go.unistack.org/micro-server-http/v3"
|
||||
pb "go.unistack.org/pkgdash/proto"
|
||||
)
|
||||
|
||||
func (h *Handler) PackageDelete(ctx context.Context, req *pb.PackageDeleteReq, rsp *pb.PackageDeleteRsp) error {
|
||||
h.logger.Debug(ctx, "Start UpdatePackage")
|
||||
|
||||
if err := req.Validate(); err != nil {
|
||||
h.logger.Error(ctx, "validate error", err)
|
||||
httpsrv.SetRspCode(ctx, http.StatusBadRequest)
|
||||
return httpsrv.SetError(NewValidationError(err))
|
||||
}
|
||||
|
||||
if err := h.store.PackageDelete(ctx, req); err != nil {
|
||||
h.logger.Error(ctx, "package delete error", err)
|
||||
httpsrv.SetRspCode(ctx, http.StatusInternalServerError)
|
||||
return httpsrv.SetError(NewInternalError(err))
|
||||
}
|
||||
|
||||
h.logger.Debug(ctx, "Success finish UpdatePackage")
|
||||
return nil
|
||||
}
|
||||
27
internal/handler/package_list.go
Normal file
27
internal/handler/package_list.go
Normal file
@@ -0,0 +1,27 @@
|
||||
package handler
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
|
||||
httpsrv "go.unistack.org/micro-server-http/v3"
|
||||
"go.unistack.org/pkgdash/internal/models"
|
||||
pb "go.unistack.org/pkgdash/proto"
|
||||
)
|
||||
|
||||
func (h *Handler) PackageList(ctx context.Context, req *pb.PackageListReq, rsp *pb.PackageListRsp) error {
|
||||
h.logger.Debug(ctx, "PackagesList handler start")
|
||||
|
||||
packages, err := h.store.PackageList(ctx, req)
|
||||
if err != nil {
|
||||
h.logger.Error(ctx, "error db response: %v", err)
|
||||
httpsrv.SetRspCode(ctx, http.StatusInternalServerError)
|
||||
return httpsrv.SetError(NewInternalError(err))
|
||||
}
|
||||
|
||||
for _, pkg := range packages {
|
||||
rsp.Packages = append(rsp.Packages, models.NewPackage(pkg))
|
||||
}
|
||||
h.logger.Debug(ctx, "PackagesList handler stop")
|
||||
return nil
|
||||
}
|
||||
32
internal/handler/package_lookup.go
Normal file
32
internal/handler/package_lookup.go
Normal file
@@ -0,0 +1,32 @@
|
||||
package handler
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
|
||||
httpsrv "go.unistack.org/micro-server-http/v3"
|
||||
"go.unistack.org/pkgdash/internal/models"
|
||||
pb "go.unistack.org/pkgdash/proto"
|
||||
)
|
||||
|
||||
func (h *Handler) PackageLookup(ctx context.Context, req *pb.PackageLookupReq, rsp *pb.PackageLookupRsp) error {
|
||||
h.logger.Debug(ctx, "Start PackagesLookup")
|
||||
|
||||
if err := req.Validate(); err != nil {
|
||||
h.logger.Error(ctx, "validate error", err)
|
||||
httpsrv.SetRspCode(ctx, http.StatusBadRequest)
|
||||
return httpsrv.SetError(NewValidationError(err))
|
||||
}
|
||||
|
||||
pkg, err := h.store.PackageLookup(ctx, req)
|
||||
if err != nil {
|
||||
h.logger.Error(ctx, "package lookup", err)
|
||||
httpsrv.SetRspCode(ctx, http.StatusInternalServerError)
|
||||
return httpsrv.SetError(NewInternalError(err))
|
||||
}
|
||||
|
||||
rsp.Package = models.NewPackage(pkg)
|
||||
|
||||
h.logger.Debug(ctx, "Success finish PackagesLookup")
|
||||
return nil
|
||||
}
|
||||
27
internal/handler/package_modules.go
Normal file
27
internal/handler/package_modules.go
Normal file
@@ -0,0 +1,27 @@
|
||||
package handler
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
|
||||
httpsrv "go.unistack.org/micro-server-http/v3"
|
||||
"go.unistack.org/pkgdash/internal/models"
|
||||
pb "go.unistack.org/pkgdash/proto"
|
||||
)
|
||||
|
||||
func (h *Handler) PackageModules(ctx context.Context, req *pb.PackageModulesReq, rsp *pb.PackageModulesRsp) error {
|
||||
h.logger.Debug(ctx, "PackageModules handler start")
|
||||
|
||||
modules, err := h.store.PackageModules(ctx, req)
|
||||
if err != nil {
|
||||
h.logger.Error(ctx, "error db response: %v", err)
|
||||
httpsrv.SetRspCode(ctx, http.StatusInternalServerError)
|
||||
return httpsrv.SetError(NewInternalError(err))
|
||||
}
|
||||
|
||||
for _, mod := range modules {
|
||||
rsp.Modules = append(rsp.Modules, models.NewModule(mod))
|
||||
}
|
||||
h.logger.Debug(ctx, "PackagesModules handler stop")
|
||||
return nil
|
||||
}
|
||||
32
internal/handler/package_update.go
Normal file
32
internal/handler/package_update.go
Normal file
@@ -0,0 +1,32 @@
|
||||
package handler
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net/http"
|
||||
|
||||
httpsrv "go.unistack.org/micro-server-http/v3"
|
||||
"go.unistack.org/pkgdash/internal/models"
|
||||
pb "go.unistack.org/pkgdash/proto"
|
||||
)
|
||||
|
||||
func (h *Handler) PackageUpdate(ctx context.Context, req *pb.PackageUpdateReq, rsp *pb.PackageUpdateRsp) error {
|
||||
h.logger.Debug(ctx, "Start UpdatePackage")
|
||||
|
||||
if err := req.Validate(); err != nil {
|
||||
h.logger.Error(ctx, "validate error", err)
|
||||
httpsrv.SetRspCode(ctx, http.StatusBadRequest)
|
||||
return httpsrv.SetError(NewValidationError(err))
|
||||
}
|
||||
|
||||
pkg, err := h.store.PackageUpdate(ctx, req)
|
||||
if err != nil {
|
||||
h.logger.Error(ctx, "package update error", err)
|
||||
httpsrv.SetRspCode(ctx, http.StatusInternalServerError)
|
||||
return httpsrv.SetError(NewInternalError(err))
|
||||
}
|
||||
|
||||
rsp.Package = models.NewPackage(pkg)
|
||||
|
||||
h.logger.Debug(ctx, "Success finish UpdatePackage")
|
||||
return nil
|
||||
}
|
||||
126
internal/models/models.go
Normal file
126
internal/models/models.go
Normal file
@@ -0,0 +1,126 @@
|
||||
package models
|
||||
|
||||
import (
|
||||
"database/sql"
|
||||
"time"
|
||||
|
||||
pb "go.unistack.org/pkgdash/proto"
|
||||
"google.golang.org/protobuf/types/known/timestamppb"
|
||||
)
|
||||
|
||||
type Handler struct {
|
||||
ID uint64 `db:"id"`
|
||||
Package uint64 `db:"package"`
|
||||
Name string `db:"name"`
|
||||
Coverage sql.NullFloat64 `db:"coverage"`
|
||||
}
|
||||
|
||||
func NewHandler(hdlr *Handler) *pb.Handler {
|
||||
if hdlr == nil {
|
||||
return nil
|
||||
}
|
||||
rsp := &pb.Handler{
|
||||
Id: hdlr.ID,
|
||||
Package: hdlr.Package,
|
||||
Name: hdlr.Name,
|
||||
}
|
||||
if hdlr.Coverage.Valid {
|
||||
rsp.Coverage = hdlr.Coverage.Float64
|
||||
}
|
||||
return rsp
|
||||
}
|
||||
|
||||
type Package struct {
|
||||
Created time.Time `db:"created"`
|
||||
Updated time.Time `db:"updated"`
|
||||
LastCheck sql.NullTime `db:"last_check"`
|
||||
Type string `db:"type"`
|
||||
Name string `db:"name"`
|
||||
URL string `db:"url"`
|
||||
Description sql.NullString `db:"description"`
|
||||
Coverage sql.NullFloat64 `db:"coverage"`
|
||||
Modules uint64 `db:"modules"`
|
||||
ID uint64 `db:"id"`
|
||||
Status uint64 `db:"status"`
|
||||
Comments uint64 `db:"comments"`
|
||||
Issues uint64 `db:"issues"`
|
||||
}
|
||||
|
||||
func NewPackage(pkg *Package) *pb.Package {
|
||||
if pkg == nil {
|
||||
return nil
|
||||
}
|
||||
rsp := &pb.Package{
|
||||
Name: pkg.Name,
|
||||
Url: pkg.URL,
|
||||
Modules: pkg.Modules,
|
||||
Issues: pkg.Issues,
|
||||
Comments: pkg.Comments,
|
||||
Id: pkg.ID,
|
||||
Created: timestamppb.New(pkg.Created),
|
||||
Updated: timestamppb.New(pkg.Updated),
|
||||
Type: pkg.Type,
|
||||
}
|
||||
if rsp.Type == "" {
|
||||
rsp.Type = "package"
|
||||
}
|
||||
if pkg.Description.Valid {
|
||||
rsp.Description = pkg.Description.String
|
||||
}
|
||||
if pkg.LastCheck.Valid {
|
||||
rsp.LastCheck = timestamppb.New(pkg.LastCheck.Time)
|
||||
}
|
||||
if pkg.Coverage.Valid {
|
||||
rsp.Coverage = pkg.Coverage.Float64
|
||||
}
|
||||
return rsp
|
||||
}
|
||||
|
||||
type Module struct {
|
||||
LastCheck sql.NullTime `db:"last_check"`
|
||||
Name string `db:"name"`
|
||||
Version string `db:"version"`
|
||||
ID uint64 `db:"id"`
|
||||
}
|
||||
|
||||
func NewModule(mod *Module) *pb.Module {
|
||||
if mod == nil {
|
||||
return nil
|
||||
}
|
||||
rsp := &pb.Module{
|
||||
Name: mod.Name,
|
||||
Version: mod.Version,
|
||||
Id: mod.ID,
|
||||
}
|
||||
if mod.LastCheck.Valid {
|
||||
rsp.LastCheck = timestamppb.New(mod.LastCheck.Time)
|
||||
}
|
||||
return rsp
|
||||
}
|
||||
|
||||
type Issue struct {
|
||||
Comment string `db:"comment"`
|
||||
Modules []int64 `db:"modules"`
|
||||
ID uint64 `db:"id"`
|
||||
Status uint64 `db:"status"`
|
||||
Package uint64 `db:"package"`
|
||||
}
|
||||
|
||||
type Comment struct {
|
||||
Created time.Time `db:"created"`
|
||||
Updated time.Time `db:"updated"`
|
||||
Comment string `db:"comment"`
|
||||
ID uint64 `db:"id"`
|
||||
}
|
||||
|
||||
func NewComment(com *Comment) *pb.Comment {
|
||||
if com == nil {
|
||||
return nil
|
||||
}
|
||||
return &pb.Comment{
|
||||
Id: com.ID,
|
||||
Comment: com.Comment,
|
||||
Created: timestamppb.New(com.Created),
|
||||
Updated: timestamppb.New(com.Updated),
|
||||
}
|
||||
}
|
||||
300
internal/modules/modproxy.go
Normal file
300
internal/modules/modproxy.go
Normal file
@@ -0,0 +1,300 @@
|
||||
package modules
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os"
|
||||
"path"
|
||||
"strconv"
|
||||
"strings"
|
||||
|
||||
"golang.org/x/mod/module"
|
||||
"golang.org/x/mod/semver"
|
||||
"golang.org/x/sync/errgroup"
|
||||
)
|
||||
|
||||
// Module contains the module path and versions
|
||||
type Module struct {
|
||||
Path string
|
||||
Versions []string
|
||||
}
|
||||
|
||||
// MaxVersion returns the latest version.
|
||||
// If there are no versions, the empty string is returned.
|
||||
// Prefix can be used to filter the versions based on a prefix.
|
||||
// If pre is false, pre-release versions will are excluded.
|
||||
func (m *Module) MaxVersion(prefix string, pre bool) string {
|
||||
var max string
|
||||
for _, v := range m.Versions {
|
||||
if !semver.IsValid(v) || !strings.HasPrefix(v, prefix) {
|
||||
continue
|
||||
}
|
||||
if !pre && semver.Prerelease(v) != "" {
|
||||
continue
|
||||
}
|
||||
max = MaxVersion(v, max)
|
||||
}
|
||||
return max
|
||||
}
|
||||
|
||||
// IsNewerVersion returns true if newversion is greater than oldversion in terms of semver.
|
||||
// If major is true, then newversion must be a major version ahead of oldversion to be considered newer.
|
||||
func IsNewerVersion(oldversion, newversion string, major bool) bool {
|
||||
if major {
|
||||
return semver.Compare(semver.Major(oldversion), semver.Major(newversion)) < 0
|
||||
}
|
||||
return semver.Compare(oldversion, newversion) < 0
|
||||
}
|
||||
|
||||
// MaxVersion returns the larger of two versions according to semantic version precedence.
|
||||
// Incompatible versions are considered lower than non-incompatible ones.
|
||||
// Invalid versions are considered lower than valid ones.
|
||||
// If both versions are invalid, the empty string is returned.
|
||||
func MaxVersion(v, w string) string {
|
||||
// sort by validity
|
||||
vValid := semver.IsValid(v)
|
||||
wValid := semver.IsValid(w)
|
||||
if !vValid && !wValid {
|
||||
return ""
|
||||
}
|
||||
if vValid != wValid {
|
||||
if vValid {
|
||||
return v
|
||||
}
|
||||
return w
|
||||
}
|
||||
// sort by compatibility
|
||||
vIncompatible := strings.HasSuffix(semver.Build(v), "+incompatible")
|
||||
wIncompatible := strings.HasSuffix(semver.Build(w), "+incompatible")
|
||||
if vIncompatible != wIncompatible {
|
||||
if wIncompatible {
|
||||
return v
|
||||
}
|
||||
return w
|
||||
}
|
||||
// sort by semver
|
||||
if semver.Compare(v, w) == 1 {
|
||||
return v
|
||||
}
|
||||
return w
|
||||
}
|
||||
|
||||
// NextMajor returns the next major version after the provided version
|
||||
func NextMajor(version string) (string, error) {
|
||||
major, err := strconv.Atoi(strings.TrimPrefix(semver.Major(version), "v"))
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
major++
|
||||
return fmt.Sprintf("v%d", major), nil
|
||||
}
|
||||
|
||||
// WithMajorPath returns the module path for the provided version
|
||||
func (m *Module) WithMajorPath(version string) string {
|
||||
prefix := ModPrefix(m.Path)
|
||||
return JoinPath(prefix, version, "")
|
||||
}
|
||||
|
||||
// NextMajorPath returns the module path of the next major version
|
||||
func (m *Module) NextMajorPath() (string, bool) {
|
||||
latest := m.MaxVersion("", true)
|
||||
if latest == "" {
|
||||
return "", false
|
||||
}
|
||||
if semver.Major(latest) == "v0" {
|
||||
return "", false
|
||||
}
|
||||
next, err := NextMajor(latest)
|
||||
if err != nil {
|
||||
return "", false
|
||||
}
|
||||
return m.WithMajorPath(next), true
|
||||
}
|
||||
|
||||
// Query the module proxy for all versions of a module.
|
||||
// If the module does not exist, the second return parameter will be false
|
||||
// cached sets the Disable-Module-Fetch: true header
|
||||
func Query(modpath string, cached bool) (*Module, bool, error) {
|
||||
escaped, err := module.EscapePath(modpath)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
url := fmt.Sprintf("https://proxy.golang.org/%s/@v/list", escaped)
|
||||
req, err := http.NewRequest(http.MethodGet, url, nil)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
req.Header.Set("User-Agent", "GoMajor/1.0")
|
||||
if cached {
|
||||
req.Header.Set("Disable-Module-Fetch", "true")
|
||||
}
|
||||
res, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
defer res.Body.Close()
|
||||
if res.StatusCode != http.StatusOK {
|
||||
body, _ := io.ReadAll(res.Body)
|
||||
if res.StatusCode == http.StatusNotFound && bytes.HasPrefix(body, []byte("not found:")) {
|
||||
return nil, false, nil
|
||||
}
|
||||
msg := string(body)
|
||||
if msg == "" {
|
||||
msg = res.Status
|
||||
}
|
||||
return nil, false, fmt.Errorf("proxy: %s", msg)
|
||||
}
|
||||
var mod Module
|
||||
mod.Path = modpath
|
||||
sc := bufio.NewScanner(res.Body)
|
||||
for sc.Scan() {
|
||||
mod.Versions = append(mod.Versions, sc.Text())
|
||||
}
|
||||
if err := sc.Err(); err != nil {
|
||||
return nil, false, err
|
||||
}
|
||||
return &mod, true, nil
|
||||
}
|
||||
|
||||
// Latest finds the latest major version of a module
|
||||
// cached sets the Disable-Module-Fetch: true header
|
||||
func Latest(modpath string, cached bool) (*Module, error) {
|
||||
latest, ok, err := Query(modpath, cached)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !ok {
|
||||
return nil, fmt.Errorf("module not found: %s", modpath)
|
||||
}
|
||||
for i := 0; i < 100; i++ {
|
||||
nextpath, ok := latest.NextMajorPath()
|
||||
if !ok {
|
||||
return latest, nil
|
||||
}
|
||||
next, ok, err := Query(nextpath, cached)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if !ok {
|
||||
// handle the case where a project switched to modules
|
||||
// without incrementing the major version
|
||||
version := latest.MaxVersion("", true)
|
||||
if semver.Build(version) == "+incompatible" {
|
||||
nextpath = latest.WithMajorPath(semver.Major(version))
|
||||
if nextpath != latest.Path {
|
||||
next, ok, err = Query(nextpath, cached)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
if !ok {
|
||||
return latest, nil
|
||||
}
|
||||
latest = next
|
||||
}
|
||||
return nil, fmt.Errorf("request limit exceeded")
|
||||
}
|
||||
|
||||
// QueryPackage tries to find the module path for the provided package path
|
||||
// it does so by repeatedly chopping off the last path element and trying to
|
||||
// use it as a path.
|
||||
func QueryPackage(pkgpath string, cached bool) (*Module, error) {
|
||||
prefix := pkgpath
|
||||
for prefix != "" {
|
||||
if module.CheckPath(prefix) == nil {
|
||||
mod, ok, err := Query(prefix, cached)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if ok {
|
||||
modprefix := ModPrefix(mod.Path)
|
||||
if modpath, pkgdir, ok := SplitPath(modprefix, pkgpath); ok && modpath != mod.Path {
|
||||
if major, ok := ModMajor(modpath); ok {
|
||||
if v := mod.MaxVersion(major, false); v != "" {
|
||||
spec := JoinPath(modprefix, "", pkgdir) + "@" + v
|
||||
return nil, fmt.Errorf("%s doesn't support import versioning; use %s", major, spec)
|
||||
}
|
||||
return nil, fmt.Errorf("failed to find module for package: %s", pkgpath)
|
||||
}
|
||||
}
|
||||
return mod, nil
|
||||
}
|
||||
}
|
||||
remaining, last := path.Split(prefix)
|
||||
if last == "" {
|
||||
break
|
||||
}
|
||||
prefix = strings.TrimSuffix(remaining, "/")
|
||||
}
|
||||
return nil, fmt.Errorf("failed to find module for package: %s", pkgpath)
|
||||
}
|
||||
|
||||
// Update reports a newer version of a module.
|
||||
// The Err field will be set if an error occured.
|
||||
type Update struct {
|
||||
Err error
|
||||
Module module.Version
|
||||
Version string
|
||||
}
|
||||
|
||||
// UpdateOptions specifies a set of modules to check for updates.
|
||||
// The OnUpdate callback will be invoked with any updates found.
|
||||
type UpdateOptions struct {
|
||||
OnUpdate func(Update)
|
||||
Modules []module.Version
|
||||
Pre bool
|
||||
Cached bool
|
||||
Major bool // Major true compare only major
|
||||
UpMajor bool // UpMajor module up with major
|
||||
}
|
||||
|
||||
// Updates finds updates for a set of specified modules.
|
||||
func Updates(opt UpdateOptions) {
|
||||
ch := make(chan Update)
|
||||
go func() {
|
||||
defer close(ch)
|
||||
private := os.Getenv("GOPRIVATE")
|
||||
var group errgroup.Group
|
||||
if opt.Cached {
|
||||
group.SetLimit(3)
|
||||
} else {
|
||||
group.SetLimit(1)
|
||||
}
|
||||
for _, m := range opt.Modules {
|
||||
m := m
|
||||
if module.MatchPrefixPatterns(private, m.Path) {
|
||||
continue
|
||||
}
|
||||
group.Go(func() error {
|
||||
mod, err := Latest(m.Path, opt.Cached)
|
||||
if err != nil {
|
||||
ch <- Update{Module: m, Err: err}
|
||||
return nil
|
||||
}
|
||||
major := semver.Major(m.Version)
|
||||
var v string
|
||||
switch opt.UpMajor {
|
||||
case true:
|
||||
v = mod.MaxVersion("", opt.Pre)
|
||||
case false:
|
||||
v = mod.MaxVersion(major, opt.Pre)
|
||||
}
|
||||
if IsNewerVersion(m.Version, v, opt.Major) {
|
||||
ch <- Update{Module: m, Version: v}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
}
|
||||
_ = group.Wait()
|
||||
}()
|
||||
for u := range ch {
|
||||
if opt.OnUpdate != nil {
|
||||
opt.OnUpdate(u)
|
||||
}
|
||||
}
|
||||
}
|
||||
115
internal/modules/packages.go
Normal file
115
internal/modules/packages.go
Normal file
@@ -0,0 +1,115 @@
|
||||
package modules
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"strings"
|
||||
|
||||
"golang.org/x/mod/module"
|
||||
"golang.org/x/mod/semver"
|
||||
)
|
||||
|
||||
// ModPrefix returns the module path with no SIV
|
||||
func ModPrefix(modpath string) string {
|
||||
prefix, _, ok := module.SplitPathVersion(modpath)
|
||||
if !ok {
|
||||
prefix = modpath
|
||||
}
|
||||
return prefix
|
||||
}
|
||||
|
||||
// ModMajor returns the major version in vN format
|
||||
func ModMajor(modpath string) (string, bool) {
|
||||
_, major, ok := module.SplitPathVersion(modpath)
|
||||
if ok {
|
||||
major = strings.TrimPrefix(major, "/")
|
||||
major = strings.TrimPrefix(major, ".")
|
||||
}
|
||||
return major, ok
|
||||
}
|
||||
|
||||
// SplitPath splits the package path into the module path and the package subdirectory.
|
||||
// It requires the a module path prefix to figure this out.
|
||||
func SplitPath(modprefix, pkgpath string) (modpath, pkgdir string, ok bool) {
|
||||
if !strings.HasPrefix(pkgpath, modprefix) {
|
||||
return "", "", false
|
||||
}
|
||||
modpathlen := len(modprefix)
|
||||
if rest := pkgpath[modpathlen:]; len(rest) > 0 && rest[0] != '/' && rest[0] != '.' {
|
||||
return "", "", false
|
||||
}
|
||||
if strings.HasPrefix(pkgpath[modpathlen:], "/") {
|
||||
modpathlen++
|
||||
}
|
||||
if idx := strings.Index(pkgpath[modpathlen:], "/"); idx >= 0 {
|
||||
modpathlen += idx
|
||||
} else {
|
||||
modpathlen = len(pkgpath)
|
||||
}
|
||||
modpath = modprefix
|
||||
if major, ok := ModMajor(pkgpath[:modpathlen]); ok {
|
||||
modpath = JoinPath(modprefix, major, "")
|
||||
}
|
||||
pkgdir = strings.TrimPrefix(pkgpath[len(modpath):], "/")
|
||||
return modpath, pkgdir, true
|
||||
}
|
||||
|
||||
// SplitSpec splits the path/to/package@query format strings
|
||||
func SplitSpec(spec string) (path, query string) {
|
||||
parts := strings.SplitN(spec, "@", 2)
|
||||
if len(parts) == 2 {
|
||||
path = parts[0]
|
||||
query = parts[1]
|
||||
} else {
|
||||
path = spec
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// JoinPath creates a full package path given a module prefix, version, and package directory.
|
||||
func JoinPath(modprefix, version, pkgdir string) string {
|
||||
version = strings.TrimPrefix(version, ".")
|
||||
version = strings.TrimPrefix(version, "/")
|
||||
major := semver.Major(version)
|
||||
pkgpath := modprefix
|
||||
switch {
|
||||
case strings.HasPrefix(pkgpath, "gopkg.in"):
|
||||
pkgpath += "." + major
|
||||
case major != "" && major != "v0" && major != "v1" && !strings.Contains(version, "+incompatible"):
|
||||
if !strings.HasSuffix(pkgpath, "/") {
|
||||
pkgpath += "/"
|
||||
}
|
||||
pkgpath += major
|
||||
}
|
||||
if pkgdir != "" {
|
||||
pkgpath += "/" + pkgdir
|
||||
}
|
||||
return pkgpath
|
||||
}
|
||||
|
||||
// FindModFile recursively searches up the directory structure until it
|
||||
// finds the go.mod, reaches the root of the directory tree, or encounters
|
||||
// an error.
|
||||
func FindModFile(dir string) (string, error) {
|
||||
var err error
|
||||
dir, err = filepath.Abs(dir)
|
||||
if err != nil {
|
||||
return "", err
|
||||
}
|
||||
for {
|
||||
name := filepath.Join(dir, "go.mod")
|
||||
_, err := os.Stat(name)
|
||||
if err == nil {
|
||||
return name, nil
|
||||
}
|
||||
if !os.IsNotExist(err) {
|
||||
return "", err
|
||||
}
|
||||
parent := filepath.Dir(dir)
|
||||
if parent == dir {
|
||||
return "", fmt.Errorf("cannot find go.mod")
|
||||
}
|
||||
dir = parent
|
||||
}
|
||||
}
|
||||
55
internal/modules/packages_test.go
Normal file
55
internal/modules/packages_test.go
Normal file
@@ -0,0 +1,55 @@
|
||||
package modules
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestModMajor(t *testing.T) {
|
||||
type args struct {
|
||||
modpath string
|
||||
}
|
||||
var tests = []struct {
|
||||
name string
|
||||
args args
|
||||
want string
|
||||
want1 bool
|
||||
}{
|
||||
{"Test #1",
|
||||
args{
|
||||
"github.com/jackc/chunkreader/v2",
|
||||
},
|
||||
"v2",
|
||||
true,
|
||||
},
|
||||
{"Test #2",
|
||||
args{
|
||||
"github.com/jackc/chunkreader",
|
||||
},
|
||||
"",
|
||||
true,
|
||||
},
|
||||
{"Test #3",
|
||||
args{
|
||||
"gopkg.in/yaml.v2",
|
||||
},
|
||||
"v2",
|
||||
true,
|
||||
},
|
||||
{"Test #4",
|
||||
args{
|
||||
"github.com/jackc/chunkreader/v1",
|
||||
},
|
||||
"",
|
||||
false,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, got1 := ModMajor(tt.args.modpath)
|
||||
if got != tt.want {
|
||||
t.Error("ModMajor() got = %v, want %v", got, tt.want)
|
||||
}
|
||||
if got1 != tt.want1 {
|
||||
t.Error("ModMajor() got1 = %v, want %v", got1, tt.want1)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
48
internal/source/git/gogit.go
Normal file
48
internal/source/git/gogit.go
Normal file
@@ -0,0 +1,48 @@
|
||||
//go:build gogit
|
||||
|
||||
package git
|
||||
|
||||
/*
|
||||
import "context"
|
||||
|
||||
type Repository interface {
|
||||
Checkout(ctx context.Context, hash string) error
|
||||
}
|
||||
|
||||
type repository struct {
|
||||
path string
|
||||
}
|
||||
|
||||
func NewRepositoryFromURL(ctx context.Context, url string) (Repository, error) {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
|
||||
Branches() {
|
||||
refIter, err := repo.Branches() // получение веток
|
||||
if err != nil {
|
||||
g.logger.Error(ctx, "failed to get branches", err)
|
||||
return err
|
||||
}
|
||||
|
||||
for {
|
||||
ref, err := refIter.Next()
|
||||
if err != nil {
|
||||
if err == io.EOF {
|
||||
break
|
||||
}
|
||||
g.logger.Error(ctx, "ref iter error", err)
|
||||
return err
|
||||
}
|
||||
g.logger.Info(ctx, fmt.Sprintf("check %s == %s", ref.Name().Short(), branch))
|
||||
if ref.Name().Short() == branch {
|
||||
headRef = plumbing.NewHashReference(ref.Name(), ref.Hash())
|
||||
g.logger.Info(ctx, "headRef set to "+headRef.String())
|
||||
break
|
||||
}
|
||||
} // перебираем получение ветки и когда находим нужную выходим из цикла записав ветку в headRef
|
||||
|
||||
refIter.Close()
|
||||
}
|
||||
|
||||
*/
|
||||
262
internal/source/git/nogogit.go
Normal file
262
internal/source/git/nogogit.go
Normal file
@@ -0,0 +1,262 @@
|
||||
//go:build !gogit
|
||||
|
||||
package git
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"fmt"
|
||||
"io"
|
||||
"os/exec"
|
||||
"strings"
|
||||
|
||||
"github.com/go-git/go-git/v5"
|
||||
"github.com/go-git/go-git/v5/plumbing"
|
||||
)
|
||||
|
||||
type Repository interface {
|
||||
Branches() ([]*plumbing.Reference, error)
|
||||
// Auth(username string, password string) error
|
||||
FetchContext(ctx context.Context, opts *git.FetchOptions) error
|
||||
PushContext(ctx context.Context, opts *git.PushOptions) error
|
||||
Head() (*plumbing.Reference, error)
|
||||
Worktree() (Worktree, error)
|
||||
}
|
||||
|
||||
type Worktree interface {
|
||||
Checkout(*git.CheckoutOptions) error
|
||||
PullContext(ctx context.Context, opts *git.PullOptions) error
|
||||
Status() (git.Status, error)
|
||||
AddWithOptions(opts *git.AddOptions) error
|
||||
Commit(msg string, opts *git.CommitOptions) (plumbing.Hash, error)
|
||||
Reset(opts *git.ResetOptions) error
|
||||
}
|
||||
|
||||
type repository struct {
|
||||
gocmd string
|
||||
path string
|
||||
// authUsername string
|
||||
// authPassword string
|
||||
}
|
||||
|
||||
func PlainOpenWithOptions(path string, opts *git.PlainOpenOptions) (Repository, error) {
|
||||
gopath, err := exec.LookPath("git")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &repository{path: path, gocmd: gopath}, nil
|
||||
}
|
||||
|
||||
/*
|
||||
func (r *repository) Auth(username string, password string) error {
|
||||
r.authUsername = username
|
||||
r.authPassword = password
|
||||
return nil
|
||||
}
|
||||
*/
|
||||
|
||||
func (r *repository) Branches() ([]*plumbing.Reference, error) {
|
||||
var branches []*plumbing.Reference
|
||||
cmd := exec.Command(r.gocmd, "show-ref", "--branches")
|
||||
buf, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("output %s error %w", buf, err)
|
||||
}
|
||||
br := bytes.NewBuffer(buf)
|
||||
for {
|
||||
line, err := br.ReadString('\n')
|
||||
if err != nil {
|
||||
if err == io.EOF && line == "" {
|
||||
break
|
||||
} else if err != io.EOF && line == "" {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
fields := strings.Fields(line)
|
||||
if len(fields) != 2 {
|
||||
return nil, fmt.Errorf("invalid fields %s", line)
|
||||
}
|
||||
branches = append(branches, plumbing.NewReferenceFromStrings(fields[1], fields[0]))
|
||||
}
|
||||
return branches, nil
|
||||
}
|
||||
|
||||
func (r *repository) FetchContext(ctx context.Context, opts *git.FetchOptions) error {
|
||||
args := []string{"fetch"}
|
||||
if opts.Force {
|
||||
args = append(args, "-f")
|
||||
}
|
||||
cmd := exec.CommandContext(ctx, r.gocmd, args...)
|
||||
buf, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
return fmt.Errorf("output %s error %w", buf, err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *repository) PushContext(ctx context.Context, opts *git.PushOptions) error {
|
||||
args := []string{"push"}
|
||||
|
||||
if opts.Force {
|
||||
args = append(args, "-f")
|
||||
}
|
||||
|
||||
/* TODO
|
||||
var refs []string
|
||||
for _, ref := range opts.RefSpecs {
|
||||
refs = append(refs, ref.String())
|
||||
}
|
||||
|
||||
args = append(args, strings.Join(refs, " "))
|
||||
*/
|
||||
cmd := exec.CommandContext(ctx, r.gocmd, args...)
|
||||
buf, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
return fmt.Errorf("output %s error %w", buf, err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (r *repository) Head() (*plumbing.Reference, error) {
|
||||
var head *plumbing.Reference
|
||||
cmd := exec.Command(r.gocmd, "symbolic-ref", "--short", "HEAD")
|
||||
buf, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("output %s error %w", buf, err)
|
||||
}
|
||||
br := bytes.NewBuffer(buf)
|
||||
for {
|
||||
line, err := br.ReadString('\n')
|
||||
if err != nil {
|
||||
if err == io.EOF && line == "" {
|
||||
break
|
||||
} else if err != io.EOF && line == "" {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
fields := strings.Fields(line)
|
||||
if len(fields) != 2 {
|
||||
return nil, fmt.Errorf("invalid fields %s", line)
|
||||
}
|
||||
head = plumbing.NewReferenceFromStrings("HEAD", fields[0])
|
||||
}
|
||||
return head, nil
|
||||
}
|
||||
|
||||
type worktree struct {
|
||||
gocmd string
|
||||
}
|
||||
|
||||
func (r *repository) Worktree() (Worktree, error) {
|
||||
return &worktree{gocmd: r.gocmd}, nil
|
||||
}
|
||||
|
||||
func (w *worktree) Checkout(opts *git.CheckoutOptions) error {
|
||||
args := []string{"checkout"}
|
||||
if opts.Create {
|
||||
args = append(args, "-b", opts.Branch.Short())
|
||||
}
|
||||
if opts.Force {
|
||||
args = append(args, "-f")
|
||||
}
|
||||
if opts.Hash.IsZero() {
|
||||
args = append(args, opts.Branch.Short())
|
||||
} else {
|
||||
args = append(args, opts.Hash.String())
|
||||
}
|
||||
cmd := exec.Command(w.gocmd, args...)
|
||||
buf, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
return fmt.Errorf("output %s error %w", buf, err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (w *worktree) Status() (git.Status, error) {
|
||||
return git.Status{}, nil
|
||||
}
|
||||
|
||||
func (w *worktree) Reset(opts *git.ResetOptions) error {
|
||||
args := []string{"reset"}
|
||||
if opts.Mode == git.HardReset {
|
||||
args = append(args, "--hard")
|
||||
}
|
||||
|
||||
args = append(args, opts.Commit.String())
|
||||
|
||||
cmd := exec.Command(w.gocmd, args...)
|
||||
buf, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
_ = buf
|
||||
return nil
|
||||
}
|
||||
|
||||
func (w *worktree) Commit(msg string, opts *git.CommitOptions) (plumbing.Hash, error) {
|
||||
cmd := exec.Command(w.gocmd, `commit`,
|
||||
fmt.Sprintf(`--author="%s <%s>"`, opts.Author.Name, opts.Author.Email),
|
||||
"-m", msg,
|
||||
fmt.Sprintf(`--date="%s"`, opts.Author.When.Format(`Mon Jan _2 15:04:05 2006 -0700`)),
|
||||
)
|
||||
buf, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
return plumbing.ZeroHash, fmt.Errorf("output %s error %w", buf, err)
|
||||
}
|
||||
|
||||
var head *plumbing.Reference
|
||||
cmd = exec.Command(w.gocmd, "show-ref", "HEAD")
|
||||
buf, err = cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
return plumbing.ZeroHash, err
|
||||
}
|
||||
br := bytes.NewBuffer(buf)
|
||||
for {
|
||||
line, err := br.ReadString('\n')
|
||||
if err != nil {
|
||||
if err == io.EOF && line == "" {
|
||||
break
|
||||
} else if err != io.EOF && line == "" {
|
||||
return plumbing.ZeroHash, err
|
||||
}
|
||||
}
|
||||
fields := strings.Fields(line)
|
||||
if len(fields) != 2 {
|
||||
return plumbing.ZeroHash, fmt.Errorf("invalid fields %s", line)
|
||||
}
|
||||
head = plumbing.NewReferenceFromStrings("HEAD", fields[0])
|
||||
}
|
||||
|
||||
return head.Hash(), nil
|
||||
}
|
||||
|
||||
func (w *worktree) PullContext(ctx context.Context, opts *git.PullOptions) error {
|
||||
args := []string{"pull"}
|
||||
if opts.Force {
|
||||
args = append(args, "-f")
|
||||
}
|
||||
if opts.Depth != 0 {
|
||||
args = append(args, fmt.Sprintf("--depth=%d", opts.Depth))
|
||||
}
|
||||
|
||||
cmd := exec.CommandContext(ctx, w.gocmd, args...)
|
||||
buf, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
return fmt.Errorf("output %s error %w", buf, err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (w *worktree) AddWithOptions(opts *git.AddOptions) error {
|
||||
cmd := exec.Command(w.gocmd, "add", opts.Path)
|
||||
buf, err := cmd.CombinedOutput()
|
||||
if err != nil {
|
||||
return fmt.Errorf("output %s error %w", buf, err)
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
676
internal/source/gitea/gitea.go
Normal file
676
internal/source/gitea/gitea.go
Normal file
@@ -0,0 +1,676 @@
|
||||
package gitea
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os/exec"
|
||||
"regexp"
|
||||
"strings"
|
||||
"text/template"
|
||||
"time"
|
||||
|
||||
"github.com/go-git/go-git/v5"
|
||||
gitconfig "github.com/go-git/go-git/v5/config"
|
||||
"github.com/go-git/go-git/v5/plumbing"
|
||||
"github.com/go-git/go-git/v5/plumbing/object"
|
||||
httpauth "github.com/go-git/go-git/v5/plumbing/transport/http"
|
||||
"go.unistack.org/micro/v3/logger"
|
||||
"go.unistack.org/pkgdash/internal/configcli"
|
||||
"go.unistack.org/pkgdash/internal/modules"
|
||||
//gogit "go.unistack.org/pkgdash/internal/source/git"
|
||||
)
|
||||
|
||||
var ErrPRNotExist = errors.New("pull request does not exist")
|
||||
|
||||
type Gitea struct {
|
||||
logger logger.Logger
|
||||
URL string
|
||||
Username string
|
||||
Password string
|
||||
PRTitle string
|
||||
PRBody string
|
||||
Repository string
|
||||
Owner string
|
||||
pulls []*giteaPull
|
||||
}
|
||||
|
||||
func NewGitea(cfg configcli.Config, log logger.Logger) *Gitea {
|
||||
return &Gitea{
|
||||
logger: log,
|
||||
URL: cfg.Source.APIURL,
|
||||
Username: cfg.Source.Username,
|
||||
Password: cfg.Source.Password,
|
||||
PRTitle: cfg.PullRequestTitle,
|
||||
PRBody: cfg.PullRequestBody,
|
||||
Repository: cfg.Source.Repository,
|
||||
Owner: cfg.Source.Owner,
|
||||
}
|
||||
}
|
||||
|
||||
type giteaPull struct {
|
||||
URL string `json:"url"`
|
||||
Title string `json:"title"`
|
||||
Base struct {
|
||||
Ref string `json:"ref"`
|
||||
} `json:"base"`
|
||||
Head struct {
|
||||
Ref string `json:"ref"`
|
||||
} `json:"head"`
|
||||
ID int64 `json:"id"`
|
||||
}
|
||||
|
||||
func (g *Gitea) Name() string {
|
||||
return "gitea"
|
||||
}
|
||||
|
||||
func (g *Gitea) RequestOpen(ctx context.Context, branch string, path string, mod modules.Update) error {
|
||||
g.logger.Debug(ctx, fmt.Sprintf("RequestOpen start, mod title: %s", path))
|
||||
|
||||
var buf []byte
|
||||
var err error
|
||||
// создания шаблона названия для пулл реквеста
|
||||
tplTitle, err := template.New("pull_request_title").Parse(g.PRTitle)
|
||||
if err != nil {
|
||||
g.logger.Fatal(ctx, fmt.Sprintf("failed to parse template: %v", err))
|
||||
}
|
||||
|
||||
wTitle := bytes.NewBuffer(nil)
|
||||
// создания шаблона тела для пулл реквеста
|
||||
tplBody, err := template.New("pull_request_body").Parse(g.PRTitle)
|
||||
if err != nil {
|
||||
g.logger.Fatal(ctx, fmt.Sprintf("failed to parse template: %v", err))
|
||||
}
|
||||
|
||||
wBody := bytes.NewBuffer(nil)
|
||||
|
||||
data := map[string]string{
|
||||
"Name": path,
|
||||
"VersionOld": mod.Module.Version,
|
||||
"VersionNew": mod.Version,
|
||||
}
|
||||
|
||||
if err = tplTitle.Execute(wTitle, data); err != nil {
|
||||
g.logger.Error(ctx, "failed to execute template", err)
|
||||
return err
|
||||
}
|
||||
if err = tplBody.Execute(wBody, data); err != nil {
|
||||
g.logger.Error(ctx, "failed to execute template", err)
|
||||
return err
|
||||
}
|
||||
|
||||
// открытие гит репозитория с опцией обхода репозитория для нахождения .git
|
||||
repo, err := git.PlainOpenWithOptions(".", &git.PlainOpenOptions{DetectDotGit: true})
|
||||
if err != nil {
|
||||
g.logger.Fatal(ctx, fmt.Sprintf("failed to open repo: %v", err))
|
||||
}
|
||||
|
||||
wtree, headRef, err := g.fetchCheckout(ctx, repo, branch, path, mod)
|
||||
if err != nil {
|
||||
g.logger.Error(ctx, "failed to checkout", err)
|
||||
}
|
||||
|
||||
defer func() {
|
||||
_ = g.checkout(wtree, headRef)
|
||||
}()
|
||||
|
||||
if err = g.scopeUpdateDep(ctx, path, mod); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
g.logger.Info(ctx, "worktree add go.mod")
|
||||
if err = wtree.AddWithOptions(&git.AddOptions{Path: "go.mod"}); err != nil {
|
||||
g.logger.Error(ctx, fmt.Sprintf("failed to add file: %v", err))
|
||||
return err
|
||||
}
|
||||
|
||||
g.logger.Info(ctx, "worktree add go.sum")
|
||||
if err = wtree.AddWithOptions(&git.AddOptions{Path: "go.sum"}); err != nil {
|
||||
g.logger.Error(ctx, fmt.Sprintf("failed to add file: %v", err))
|
||||
return err
|
||||
}
|
||||
|
||||
g.logger.Info(ctx, "worktree commit")
|
||||
_, err = wtree.Commit(wTitle.String(), &git.CommitOptions{
|
||||
Parents: []plumbing.Hash{headRef.Hash()},
|
||||
Author: &object.Signature{
|
||||
Name: "gitea-actions",
|
||||
Email: "info@unistack.org",
|
||||
When: time.Now(),
|
||||
},
|
||||
}) // хотим за коммитить изменения
|
||||
if err != nil {
|
||||
g.logger.Error(ctx, fmt.Sprintf("failed to commit: %v", err))
|
||||
return err
|
||||
}
|
||||
|
||||
refspec := gitconfig.RefSpec(fmt.Sprintf("+refs/heads/pkgdash/go_modules/%s-%s:refs/heads/pkgdash/go_modules/%s-%s", path, mod.Version, path, mod.Version)) // todo как будто нужно переделать
|
||||
|
||||
g.logger.Info(ctx, fmt.Sprintf("try to push refspec %s", refspec))
|
||||
|
||||
if err = repo.PushContext(ctx, &git.PushOptions{
|
||||
RefSpecs: []gitconfig.RefSpec{refspec},
|
||||
Auth: &httpauth.BasicAuth{Username: g.Username, Password: g.Password},
|
||||
Force: true,
|
||||
}); err != nil {
|
||||
g.logger.Error(ctx, "failed to push repo branch", err)
|
||||
return err
|
||||
} // пытаемся за пушить изменения
|
||||
|
||||
rsp, err := g.postPullRequest(ctx, wBody, wTitle, branch, path, mod)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// Вроде создаем новый реквест на создание пулл реквеста
|
||||
if rsp.StatusCode != http.StatusCreated {
|
||||
buf, _ = io.ReadAll(rsp.Body)
|
||||
return fmt.Errorf("unknown error: %s", buf)
|
||||
}
|
||||
|
||||
g.logger.Info(ctx, fmt.Sprintf("PR create for %s-%s", path, mod.Version))
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (g *Gitea) RequestClose(ctx context.Context, branch string, path string) error {
|
||||
g.logger.Debug(ctx, fmt.Sprintf("RequestClose start, mod title: %s", path))
|
||||
var err error
|
||||
|
||||
g.pulls, err = g.GetPulls(ctx, g.URL, g.Owner, g.Repository, g.Password)
|
||||
if err != nil {
|
||||
g.logger.Error(ctx, fmt.Sprintf("GetPulls error: %s", err))
|
||||
return err
|
||||
}
|
||||
|
||||
prExist := false
|
||||
var b string // Name of the branch to be deleted
|
||||
for _, pull := range g.pulls {
|
||||
if strings.Contains(pull.Title, path) && pull.Base.Ref == branch {
|
||||
g.logger.Info(ctx, fmt.Sprintf("PR for %s exists: %s", path, pull.URL))
|
||||
prExist = true
|
||||
b = pull.Head.Ref
|
||||
}
|
||||
}
|
||||
if !prExist {
|
||||
g.logger.Error(ctx, fmt.Sprintf("skip %s since pr does not exist", path))
|
||||
return ErrPRNotExist
|
||||
}
|
||||
|
||||
req, err := g.DeleteBranch(ctx, g.URL, g.Owner, g.Repository, b, g.Password)
|
||||
if err != nil {
|
||||
g.logger.Error(ctx, fmt.Sprintf("failed to create request for delete the branch: %s, err: %s", branch, err))
|
||||
return err
|
||||
}
|
||||
rsp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
g.logger.Error(ctx, fmt.Sprintf("failed to do request for delete the branch: %s, err: %s, code: %v", branch, err, rsp.StatusCode))
|
||||
return err
|
||||
}
|
||||
|
||||
g.logger.Info(ctx, fmt.Sprintf("Delete branch for %s successful", path))
|
||||
return nil
|
||||
}
|
||||
|
||||
func (g *Gitea) RequestUpdate(ctx context.Context, branch string, path string, mod modules.Update) error {
|
||||
g.logger.Debug(ctx, fmt.Sprintf("RequestUpdate start, mod title: %s", path))
|
||||
var err error
|
||||
|
||||
g.pulls, err = g.GetPulls(ctx, g.URL, g.Owner, g.Repository, g.Password)
|
||||
if err != nil {
|
||||
g.logger.Error(ctx, fmt.Sprintf("GetPulls error: %s", err))
|
||||
return err
|
||||
}
|
||||
|
||||
prExist := false
|
||||
var pullId int64
|
||||
var targetBranch plumbing.ReferenceName
|
||||
|
||||
for _, pull := range g.pulls {
|
||||
if strings.Contains(pull.Title, path) && pull.Base.Ref == branch {
|
||||
g.logger.Info(ctx, fmt.Sprintf("don't skip %s since pr exist %s", path, pull.URL)) // todo
|
||||
tVersion := getVersions(pull.Head.Ref) // Надо взять просто из названия ветки последнюю версию
|
||||
if !modules.IsNewerVersion(tVersion, mod.Version, false) {
|
||||
g.logger.Debug(ctx, "The existing PR is relevant")
|
||||
return nil
|
||||
}
|
||||
prExist = true
|
||||
pullId = pull.ID
|
||||
targetBranch = plumbing.ReferenceName(pull.Head.Ref)
|
||||
}
|
||||
}
|
||||
if !prExist {
|
||||
g.logger.Error(ctx, fmt.Sprintf("skip %s since pr does not exist", path))
|
||||
return ErrPRNotExist
|
||||
}
|
||||
|
||||
// создания шаблона названия для пулл реквеста
|
||||
tplTitle, err := template.New("pull_request_title").Parse(g.PRTitle)
|
||||
if err != nil {
|
||||
g.logger.Fatal(ctx, fmt.Sprintf("failed to parse template: %v", err))
|
||||
}
|
||||
|
||||
wTitle := bytes.NewBuffer(nil)
|
||||
// создания шаблона тела для пулл реквеста
|
||||
tplBody, err := template.New("pull_request_body").Parse(g.PRTitle)
|
||||
if err != nil {
|
||||
g.logger.Fatal(ctx, fmt.Sprintf("failed to parse template: %v", err))
|
||||
}
|
||||
|
||||
wBody := bytes.NewBuffer(nil)
|
||||
|
||||
data := map[string]string{
|
||||
"Name": path,
|
||||
"VersionOld": mod.Module.Version,
|
||||
"VersionNew": mod.Version,
|
||||
}
|
||||
|
||||
if err = tplTitle.Execute(wTitle, data); err != nil {
|
||||
g.logger.Error(ctx, "failed to execute template", err)
|
||||
return err
|
||||
}
|
||||
if err = tplBody.Execute(wBody, data); err != nil {
|
||||
g.logger.Error(ctx, "failed to execute template", err)
|
||||
return err
|
||||
}
|
||||
|
||||
// открытие гит репозитория с опцией обхода репозитория для нахождения .git
|
||||
repo, err := git.PlainOpenWithOptions(".", &git.PlainOpenOptions{DetectDotGit: true})
|
||||
if err != nil {
|
||||
g.logger.Fatal(ctx, fmt.Sprintf("failed to open repo: %v", err))
|
||||
}
|
||||
|
||||
wtree, headRef, err := g.fetchCheckout(ctx, repo, targetBranch.Short(), path, mod)
|
||||
if err != nil {
|
||||
g.logger.Error(ctx, "failed to checkout", err)
|
||||
return err
|
||||
}
|
||||
|
||||
defer func() {
|
||||
_ = g.checkout(wtree, headRef)
|
||||
}()
|
||||
|
||||
if err = g.scopeUpdateDep(ctx, path, mod); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
g.logger.Info(ctx, "worktree add go.mod")
|
||||
if err = wtree.AddWithOptions(&git.AddOptions{Path: "go.mod"}); err != nil {
|
||||
g.logger.Error(ctx, fmt.Sprintf("failed to add file: %v", err))
|
||||
return err
|
||||
}
|
||||
|
||||
g.logger.Info(ctx, "worktree add go.sum")
|
||||
if err = wtree.AddWithOptions(&git.AddOptions{Path: "go.sum"}); err != nil {
|
||||
g.logger.Error(ctx, fmt.Sprintf("failed to add file: %v", err))
|
||||
return err
|
||||
}
|
||||
|
||||
g.logger.Info(ctx, "worktree commit")
|
||||
_, err = wtree.Commit(wTitle.String(), &git.CommitOptions{
|
||||
Parents: []plumbing.Hash{headRef.Hash()},
|
||||
Author: &object.Signature{
|
||||
Name: "gitea-actions",
|
||||
Email: "info@unistack.org",
|
||||
When: time.Now(),
|
||||
},
|
||||
}) // хотим за коммитить изменения
|
||||
if err != nil {
|
||||
g.logger.Error(ctx, fmt.Sprintf("failed to commit: %v", err))
|
||||
return err
|
||||
}
|
||||
|
||||
refspec := gitconfig.RefSpec(fmt.Sprintf("+refs/heads/pkgdash/go_modules/%s-%s:refs/heads/pkgdash/go_modules/%s-%s", path, mod.Version, path, mod.Version)) // todo как будто нужно переделать
|
||||
|
||||
g.logger.Info(ctx, fmt.Sprintf("try to push refspec %s", refspec))
|
||||
|
||||
if err = repo.PushContext(ctx, &git.PushOptions{
|
||||
//RefSpecs: []gitconfig.RefSpec{refspec},
|
||||
Auth: &httpauth.BasicAuth{Username: g.Username, Password: g.Password},
|
||||
Force: true,
|
||||
}); err != nil {
|
||||
g.logger.Error(ctx, "failed to push repo branch", err)
|
||||
return err
|
||||
} // пытаемся за пушить изменения
|
||||
|
||||
err = g.patchPullRequest(ctx, wBody, wTitle, pullId)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
g.logger.Info(ctx, fmt.Sprintf("PR update for %s-%s", path, mod.Version))
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (g *Gitea) RequestList(ctx context.Context, branch string) (map[string]string, error) {
|
||||
g.logger.Debug(ctx, fmt.Sprintf("RequestList for %s", branch))
|
||||
var err error
|
||||
|
||||
g.pulls, err = g.GetPulls(ctx, g.URL, g.Owner, g.Repository, g.Password)
|
||||
if err != nil {
|
||||
g.logger.Error(ctx, fmt.Sprintf("GetPulls error: %s", err))
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var path string
|
||||
rMap := make(map[string]string)
|
||||
|
||||
for _, pull := range g.pulls {
|
||||
if !strings.HasPrefix(pull.Title, "Bump ") || pull.Base.Ref != branch { // добавляем только реквесты бота по обновлению модулей
|
||||
continue
|
||||
}
|
||||
path = strings.Split(pull.Title, " ")[1] // todo Работет только для дефолтного шаблона
|
||||
rMap[path] = pull.Title
|
||||
}
|
||||
return rMap, nil
|
||||
}
|
||||
|
||||
func getVersions(s string) string {
|
||||
re := regexp.MustCompile("[vV][0-9]+\\.[0-9]+\\.[0-9]+")
|
||||
|
||||
version := re.FindString(s)
|
||||
|
||||
return version
|
||||
}
|
||||
|
||||
func (g *Gitea) DeleteBranch(ctx context.Context, url, owner, repo, branch, password string) (*http.Request, error) {
|
||||
var buf []byte
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodDelete, fmt.Sprintf("https://%s/api/v1/repos/%s/%s/branches/%s", url, owner, repo, branch), bytes.NewReader(buf))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req.Header.Add("Accept", "application/json")
|
||||
req.Header.Add("Content-Type", "application/json")
|
||||
req.Header.Add("Authorization", "Bearer "+password)
|
||||
return req, err
|
||||
}
|
||||
|
||||
func (g *Gitea) GetPulls(ctx context.Context, url, owner, repo, password string) ([]*giteaPull, error) {
|
||||
var pullsAll []*giteaPull
|
||||
page := 1
|
||||
|
||||
for {
|
||||
pulls := make([]*giteaPull, 0, 10)
|
||||
req, err := http.NewRequestWithContext(
|
||||
ctx,
|
||||
http.MethodGet,
|
||||
fmt.Sprintf("https://%s/api/v1/repos/%s/%s/pulls?state=open&page=%v", url, owner, repo, page),
|
||||
nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
} // вроде запроса к репозиторию
|
||||
|
||||
req.Header.Add("Accept", "application/json")
|
||||
req.Header.Add("Content-Type", "application/json")
|
||||
req.Header.Add("Authorization", "Bearer "+password)
|
||||
|
||||
rsp, err := http.DefaultClient.Do(req) // выполнение запроса
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
buf, _ := io.ReadAll(rsp.Body)
|
||||
|
||||
switch rsp.StatusCode {
|
||||
case http.StatusOK:
|
||||
if err = json.Unmarshal(buf, &pulls); err != nil {
|
||||
g.logger.Error(ctx, fmt.Sprintf("failed to decode response %s err: %v", buf, err))
|
||||
return nil, err
|
||||
}
|
||||
pullsAll = append(pullsAll, pulls...)
|
||||
page++
|
||||
case http.StatusNotFound:
|
||||
g.logger.Info(ctx, fmt.Sprintf("pull-request is not exist for %s", repo))
|
||||
return nil, ErrPRNotExist
|
||||
default:
|
||||
return nil, fmt.Errorf("unknown error: %s", buf)
|
||||
}
|
||||
|
||||
if len(pulls) == 0 {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return pullsAll, nil
|
||||
}
|
||||
|
||||
func (g *Gitea) checkout(w *git.Worktree, ref *plumbing.Reference) error {
|
||||
ctx := context.Background()
|
||||
g.logger.Debug(ctx, "checkout: "+ref.String())
|
||||
|
||||
if err := w.Checkout(&git.CheckoutOptions{
|
||||
Branch: ref.Name(),
|
||||
Create: false,
|
||||
Force: true,
|
||||
Keep: false,
|
||||
}); err != nil {
|
||||
g.logger.Error(ctx, "failed to reset", err)
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
func (g Gitea) fetchCheckout(ctx context.Context, repo *git.Repository, branch, path string, mod modules.Update) (*git.Worktree, *plumbing.Reference, error) {
|
||||
// обновляем ветки
|
||||
if err := repo.FetchContext(ctx, &git.FetchOptions{
|
||||
Auth: &httpauth.BasicAuth{Username: g.Username, Password: g.Password},
|
||||
Force: true,
|
||||
}); err != nil && err != git.NoErrAlreadyUpToDate {
|
||||
g.logger.Error(ctx, "failed to fetch repo", err)
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
var headRef *plumbing.Reference
|
||||
|
||||
branches, err := repo.Branches()
|
||||
if err != nil {
|
||||
g.logger.Error(ctx, "cant get repo branch", err)
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
for {
|
||||
ref, err := branches.Next()
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
if ref.Name().Short() == branch {
|
||||
//Получаем ссылку на нужную ветку
|
||||
headRef = ref
|
||||
g.logger.Info(ctx, "headRef set to "+headRef.String())
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if headRef == nil {
|
||||
g.logger.Error(ctx, "failed to get repo branch head")
|
||||
return nil, nil, err
|
||||
} // Не получили нужную ветку
|
||||
|
||||
g.logger.Info(ctx, "repo head "+headRef.String())
|
||||
|
||||
wtree, err := repo.Worktree()
|
||||
if err != nil {
|
||||
g.logger.Error(ctx, "failed to get worktree", err)
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
if err = wtree.Reset(&git.ResetOptions{
|
||||
Mode: git.HardReset,
|
||||
}); err != nil {
|
||||
g.logger.Error(ctx, "reset work_tree error: ", err)
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
g.pulls, err = g.GetPulls(ctx, g.URL, g.Owner, g.Repository, g.Password)
|
||||
if err != nil && err != ErrPRNotExist {
|
||||
g.logger.Error(ctx, "GetPulls error", err)
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
var pullExist bool
|
||||
for _, pull := range g.pulls {
|
||||
if strings.Contains(pull.Title, path) && (strings.Contains(pull.Base.Ref, branch) || strings.Contains(pull.Head.Ref, branch)) {
|
||||
pullExist = true
|
||||
} // хотим проверить есть ли пулл реквест для этой ветки, если есть то выходим
|
||||
}
|
||||
|
||||
g.logger.Info(ctx, fmt.Sprintf("update %s from %s to %s", path, mod.Module.Version, mod.Version))
|
||||
|
||||
wstatus, err := wtree.Status()
|
||||
if err != nil {
|
||||
g.logger.Error(ctx, "failed to get worktree status", err)
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
g.logger.Info(ctx, "worktree status "+wstatus.String())
|
||||
|
||||
if err = wtree.PullContext(ctx, &git.PullOptions{
|
||||
Auth: &httpauth.BasicAuth{Username: g.Username, Password: g.Password},
|
||||
// Depth: 1,
|
||||
// RemoteURL :
|
||||
ReferenceName: headRef.Name(),
|
||||
Force: true,
|
||||
RemoteName: "origin",
|
||||
}); err != nil && err != git.NoErrAlreadyUpToDate {
|
||||
g.logger.Error(ctx, fmt.Sprintf("failed to pull repo: %v", err)) // подтягиваем изменения с удаленого репозитория
|
||||
return nil, nil, err
|
||||
}
|
||||
|
||||
g.logger.Info(ctx, fmt.Sprintf("checkout ref %s", headRef))
|
||||
|
||||
if pullExist {
|
||||
if err = wtree.Checkout(&git.CheckoutOptions{
|
||||
Branch: headRef.Name(),
|
||||
Create: false,
|
||||
Force: true,
|
||||
}); err != nil && err != git.ErrBranchExists {
|
||||
g.logger.Error(ctx, fmt.Sprintf("failed to checkout tree: %v", err))
|
||||
return nil, nil, err
|
||||
} //переходим на существующею
|
||||
} else {
|
||||
if err = wtree.Checkout(&git.CheckoutOptions{
|
||||
Hash: headRef.Hash(),
|
||||
Branch: plumbing.NewBranchReferenceName(fmt.Sprintf("pkgdash/go_modules/%s-%s", path, mod.Version)),
|
||||
Create: true,
|
||||
Force: true,
|
||||
}); err != nil && err != git.ErrBranchExists {
|
||||
g.logger.Error(ctx, fmt.Sprintf("failed to checkout tree: %v", err))
|
||||
return nil, nil, err
|
||||
} // создаем новую ветку
|
||||
}
|
||||
return wtree, headRef, nil
|
||||
}
|
||||
|
||||
func (g *Gitea) postPullRequest(ctx context.Context, wBody, wTitle *bytes.Buffer, branch, path string, mod modules.Update) (*http.Response, error) {
|
||||
body := map[string]string{
|
||||
"base": branch,
|
||||
"body": wBody.String(),
|
||||
"head": fmt.Sprintf("pkgdash/go_modules/%s-%s", path, mod.Version),
|
||||
"title": wTitle.String(),
|
||||
}
|
||||
g.logger.Info(ctx, fmt.Sprintf("raw body: %#+v", body))
|
||||
|
||||
buf, err := json.Marshal(body)
|
||||
if err != nil {
|
||||
g.logger.Error(ctx, "failed to marshal", err)
|
||||
return nil, err
|
||||
}
|
||||
|
||||
g.logger.Info(ctx, fmt.Sprintf("marshal body: %s", buf))
|
||||
req, err := http.NewRequestWithContext(
|
||||
ctx,
|
||||
http.MethodPost,
|
||||
fmt.Sprintf("https://%s/api/v1/repos/%s/%s/pulls", g.URL, g.Owner, g.Repository),
|
||||
bytes.NewReader(buf),
|
||||
)
|
||||
if err != nil {
|
||||
g.logger.Error(ctx, "http request error", err)
|
||||
return nil, err
|
||||
}
|
||||
req.Header.Add("Accept", "application/json")
|
||||
req.Header.Add("Content-Type", "application/json")
|
||||
req.Header.Add("Authorization", "Bearer "+g.Password)
|
||||
|
||||
rsp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
g.logger.Error(ctx, "failed to call http request", err)
|
||||
return rsp, err
|
||||
}
|
||||
|
||||
return rsp, nil
|
||||
}
|
||||
|
||||
func (g *Gitea) patchPullRequest(ctx context.Context, wBody, wTitle *bytes.Buffer, indexPR int64) error {
|
||||
body := map[string]string{
|
||||
"body": wBody.String(),
|
||||
"title": wTitle.String(),
|
||||
}
|
||||
g.logger.Info(ctx, fmt.Sprintf("raw body: %#+v", body))
|
||||
|
||||
buf, err := json.Marshal(body)
|
||||
if err != nil {
|
||||
g.logger.Error(ctx, "failed to marshal", err)
|
||||
return err
|
||||
}
|
||||
|
||||
g.logger.Info(ctx, fmt.Sprintf("marshal body: %s", buf))
|
||||
req, err := http.NewRequestWithContext(
|
||||
ctx,
|
||||
http.MethodPatch,
|
||||
fmt.Sprintf("https://%s/api/v1/repos/%s/%s/pulls/%d", g.URL, g.Owner, g.Repository, indexPR),
|
||||
bytes.NewReader(buf),
|
||||
)
|
||||
if err != nil {
|
||||
g.logger.Error(ctx, "http request error", err)
|
||||
return err
|
||||
}
|
||||
req.Header.Add("Accept", "application/json")
|
||||
req.Header.Add("Content-Type", "application/json")
|
||||
req.Header.Add("Authorization", "Bearer "+g.Password)
|
||||
|
||||
_, err = http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
g.logger.Error(ctx, "failed to call http request", err)
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (g *Gitea) scopeUpdateDep(ctx context.Context, path string, mod modules.Update) error {
|
||||
epath, err := exec.LookPath("go")
|
||||
if errors.Is(err, exec.ErrDot) {
|
||||
err = nil
|
||||
}
|
||||
if err != nil {
|
||||
g.logger.Fatal(ctx, fmt.Sprintf("failed to find go command: %v", err))
|
||||
} // ищем go файл
|
||||
|
||||
var cmd *exec.Cmd
|
||||
var out []byte
|
||||
|
||||
cmd = exec.CommandContext(ctx, epath, "mod", "edit", fmt.Sprintf("-droprequire=%s", mod.Module.Path))
|
||||
if out, err = cmd.CombinedOutput(); err != nil {
|
||||
g.logger.Error(ctx, fmt.Sprintf("failed to run go mod edit: %s err: %v", out, err))
|
||||
return err
|
||||
}
|
||||
|
||||
cmd = exec.CommandContext(ctx, epath, "mod", "edit", fmt.Sprintf("-require=%s@%s", path, mod.Version))
|
||||
if out, err = cmd.CombinedOutput(); err != nil {
|
||||
g.logger.Error(ctx, fmt.Sprintf("failed to run go mod edit: %s err: %v", out, err))
|
||||
return err
|
||||
} // пытаемся выполнить команду go mod edit с новой версией модуля
|
||||
|
||||
cmd = exec.CommandContext(ctx, epath, "mod", "tidy")
|
||||
if out, err = cmd.CombinedOutput(); err != nil {
|
||||
g.logger.Error(ctx, fmt.Sprintf("failed to run go mod tidy: %s err: %v", out, err))
|
||||
return err
|
||||
} // пытаемся выполнить команду go mod tidy пытаемся подтянуть новую версию модуля
|
||||
|
||||
return nil
|
||||
}
|
||||
396
internal/source/github/github.go
Normal file
396
internal/source/github/github.go
Normal file
@@ -0,0 +1,396 @@
|
||||
package github
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os/exec"
|
||||
"strings"
|
||||
"text/template"
|
||||
"time"
|
||||
|
||||
"github.com/go-git/go-git/v5"
|
||||
gitconfig "github.com/go-git/go-git/v5/config"
|
||||
"github.com/go-git/go-git/v5/plumbing"
|
||||
"github.com/go-git/go-git/v5/plumbing/object"
|
||||
httpauth "github.com/go-git/go-git/v5/plumbing/transport/http"
|
||||
"go.unistack.org/micro/v3/logger"
|
||||
"go.unistack.org/pkgdash/internal/configcli"
|
||||
"go.unistack.org/pkgdash/internal/modules"
|
||||
)
|
||||
|
||||
var ErrPRNotExist = errors.New("pull request does not exist")
|
||||
|
||||
type Github struct {
|
||||
logger logger.Logger
|
||||
URL string
|
||||
Username string
|
||||
Password string
|
||||
PRTitle string
|
||||
PRBody string
|
||||
Repository string
|
||||
Owner string
|
||||
pulls []*githubPull
|
||||
baseRef *plumbing.Reference
|
||||
}
|
||||
|
||||
func NewGithub(cfg configcli.Config, log logger.Logger) *Github {
|
||||
return &Github{
|
||||
logger: log,
|
||||
URL: cfg.Source.APIURL,
|
||||
Username: cfg.Source.Username,
|
||||
Password: cfg.Source.Password,
|
||||
PRTitle: cfg.PullRequestTitle,
|
||||
PRBody: cfg.PullRequestBody,
|
||||
Repository: cfg.Source.Repository,
|
||||
Owner: cfg.Source.Owner,
|
||||
}
|
||||
}
|
||||
|
||||
type githubPull struct {
|
||||
URL string `json:"url"`
|
||||
Title string `json:"title"`
|
||||
Base struct {
|
||||
Ref string `json:"ref"`
|
||||
} `json:"base"`
|
||||
Head struct {
|
||||
Ref string `json:"ref"`
|
||||
} `json:"head"`
|
||||
ID int64 `json:"id"`
|
||||
}
|
||||
|
||||
func (g *Github) Name() string {
|
||||
return "github"
|
||||
}
|
||||
|
||||
func (g *Github) RequestOpen(ctx context.Context, branch string, path string, mod modules.Update) error {
|
||||
g.logger.Debug(ctx, fmt.Sprintf("RequestOpen start, mod title: %s", path))
|
||||
|
||||
var buf []byte
|
||||
var err error
|
||||
// создания шаблона названия для пулл реквеста
|
||||
tplTitle, err := template.New("pull_request_title").Parse(g.PRTitle)
|
||||
if err != nil {
|
||||
g.logger.Fatal(ctx, fmt.Sprintf("failed to parse template: %v", err))
|
||||
}
|
||||
|
||||
wTitle := bytes.NewBuffer(nil)
|
||||
// создания шаблона тела для пулл реквеста
|
||||
tplBody, err := template.New("pull_request_body").Parse(g.PRTitle)
|
||||
if err != nil {
|
||||
g.logger.Fatal(ctx, fmt.Sprintf("failed to parse template: %v", err))
|
||||
}
|
||||
|
||||
wBody := bytes.NewBuffer(nil)
|
||||
|
||||
data := map[string]string{
|
||||
"Name": path,
|
||||
"VersionOld": mod.Module.Version,
|
||||
"VersionNew": mod.Version,
|
||||
}
|
||||
|
||||
if err = tplTitle.Execute(wTitle, data); err != nil {
|
||||
g.logger.Fatal(ctx, fmt.Sprintf("failed to execute template: %v", err))
|
||||
}
|
||||
if err = tplBody.Execute(wBody, data); err != nil {
|
||||
g.logger.Fatal(ctx, fmt.Sprintf("failed to execute template: %v", err))
|
||||
}
|
||||
|
||||
// открытие гит репозитория с опцией обхода репозитория для нахождения .git
|
||||
repo, err := git.PlainOpenWithOptions(".", &git.PlainOpenOptions{DetectDotGit: true})
|
||||
if err != nil {
|
||||
g.logger.Fatal(ctx, fmt.Sprintf("failed to open repo: %v", err))
|
||||
}
|
||||
// извлекаем ссылки с объектами из удаленного объекта??
|
||||
if err = repo.FetchContext(ctx, &git.FetchOptions{
|
||||
// Auth: &httpauth.BasicAuth{Username: g.Username, Password: g.Password},
|
||||
Force: true,
|
||||
}); err != nil && err != git.NoErrAlreadyUpToDate {
|
||||
g.logger.Fatal(ctx, fmt.Sprintf("failed to fetch repo : %v", err))
|
||||
} // обновляем репозиторий
|
||||
|
||||
var headRef *plumbing.Reference // вроде ссылка на гит
|
||||
|
||||
if g.baseRef == nil {
|
||||
g.baseRef, err = repo.Head()
|
||||
if err != nil {
|
||||
g.logger.Fatal(ctx, fmt.Sprintf("Error head: %s", err))
|
||||
}
|
||||
}
|
||||
|
||||
refIter, err := repo.Branches() // получение веток
|
||||
if err != nil {
|
||||
g.logger.Fatal(ctx, fmt.Sprintf("failed to get branches: %v", err))
|
||||
return err
|
||||
}
|
||||
for {
|
||||
ref, err := refIter.Next()
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
if ref.Name().Short() == branch { // todo вот тут возможно нужно переделать
|
||||
headRef = ref
|
||||
break
|
||||
}
|
||||
} // перебираем получение ветки и когда находим нужную выходим из цикла записав ветку в headRef
|
||||
refIter.Close()
|
||||
|
||||
if headRef == nil {
|
||||
g.logger.Fatal(ctx, "failed to get repo branch head")
|
||||
return err
|
||||
} // Не получили нужную ветку
|
||||
|
||||
g.logger.Info(ctx, fmt.Sprintf("repo head %s", headRef))
|
||||
|
||||
wtree, err := repo.Worktree() // todo вроде рабочее дерево не нужно
|
||||
if err != nil {
|
||||
g.logger.Fatal(ctx, fmt.Sprintf("failed to get worktree: %v", err))
|
||||
}
|
||||
defer g.checkout(*wtree, *g.baseRef)
|
||||
|
||||
g.pulls, err = g.GetPulls(ctx, g.URL, g.Owner, g.Repository, g.Password)
|
||||
if err != nil && err != ErrPRNotExist {
|
||||
g.logger.Error(ctx, fmt.Sprintf("GetPulls error: %s", err))
|
||||
return err
|
||||
}
|
||||
|
||||
for _, pull := range g.pulls {
|
||||
if strings.Contains(pull.Title, path) && strings.Contains(pull.Base.Ref, branch) {
|
||||
g.logger.Info(ctx, fmt.Sprintf("PR for %s exists %s, call RequestUpdate", path, pull.URL))
|
||||
return g.RequestUpdate(ctx, branch, path, mod)
|
||||
} // хотим проверить есть ли пулл реквест для этой ветки, если есть то выходим
|
||||
}
|
||||
|
||||
g.logger.Info(ctx, fmt.Sprintf("update %s from %s to %s", path, mod.Module.Version, mod.Version))
|
||||
|
||||
g.logger.Info(ctx, "reset worktree")
|
||||
if err = wtree.Reset(&git.ResetOptions{Commit: headRef.Hash(), Mode: git.HardReset}); err != nil {
|
||||
g.logger.Error(ctx, fmt.Sprintf("failed to reset repo branch: %v", err))
|
||||
} // вроде меняем ветку todo вроде можно удалить
|
||||
|
||||
if err = wtree.PullContext(ctx, &git.PullOptions{
|
||||
Auth: &httpauth.BasicAuth{Username: g.Username, Password: g.Password},
|
||||
Depth: 1,
|
||||
// RemoteURL :
|
||||
Force: true,
|
||||
RemoteName: "origin",
|
||||
}); err != nil && err != git.NoErrAlreadyUpToDate {
|
||||
g.logger.Error(ctx, fmt.Sprintf("failed to pull repo: %v", err)) // подтягиваем изменения с удаленого репозитория
|
||||
}
|
||||
|
||||
g.logger.Info(ctx, fmt.Sprintf("checkout ref %s", headRef))
|
||||
if err = wtree.Checkout(&git.CheckoutOptions{
|
||||
Hash: headRef.Hash(),
|
||||
Branch: plumbing.NewBranchReferenceName(fmt.Sprintf("pkgdash/go_modules/%s-%s", path, mod.Version)),
|
||||
Create: true,
|
||||
Force: true,
|
||||
}); err != nil {
|
||||
g.logger.Error(ctx, fmt.Sprintf("failed to checkout tree: %v", err))
|
||||
return err
|
||||
} // создаем новую ветку
|
||||
|
||||
epath, err := exec.LookPath("go")
|
||||
if errors.Is(err, exec.ErrDot) {
|
||||
err = nil
|
||||
}
|
||||
if err != nil {
|
||||
g.logger.Fatal(ctx, fmt.Sprintf("failed to find go command: %v", err))
|
||||
} // ищем go файл
|
||||
|
||||
var cmd *exec.Cmd
|
||||
var out []byte
|
||||
|
||||
cmd = exec.CommandContext(ctx, epath, "mod", "edit", fmt.Sprintf("-droprequire=%s", mod.Module.Path))
|
||||
if out, err = cmd.CombinedOutput(); err != nil {
|
||||
g.logger.Fatal(ctx, fmt.Sprintf("failed to run go mod edit: %s err: %v", out, err))
|
||||
}
|
||||
|
||||
cmd = exec.CommandContext(ctx, epath, "mod", "edit", fmt.Sprintf("-require=%s@%s", path, mod.Version))
|
||||
if out, err = cmd.CombinedOutput(); err != nil {
|
||||
g.logger.Fatal(ctx, fmt.Sprintf("failed to run go mod edit: %s err: %v", out, err))
|
||||
} // пытаемся выполнить команду go mod edit с новой версией модуля
|
||||
|
||||
cmd = exec.CommandContext(ctx, epath, "mod", "tidy")
|
||||
if out, err = cmd.CombinedOutput(); err != nil {
|
||||
g.logger.Fatal(ctx, fmt.Sprintf("failed to run go mod tidy: %s err: %v", out, err))
|
||||
} // пытаемся выполнить команду go mod tidy пытаемся подтянуть новую версию модуля
|
||||
|
||||
g.logger.Info(ctx, "worktree add go.mod")
|
||||
if _, err = wtree.Add("go.mod"); err != nil {
|
||||
g.logger.Fatal(ctx, fmt.Sprintf("failed to add file: %v", err))
|
||||
}
|
||||
|
||||
g.logger.Info(ctx, "worktree add go.sum")
|
||||
if _, err = wtree.Add("go.sum"); err != nil {
|
||||
g.logger.Fatal(ctx, fmt.Sprintf("failed to add file: %v", err))
|
||||
}
|
||||
|
||||
g.logger.Info(ctx, "worktree commit")
|
||||
_, err = wtree.Commit(wTitle.String(), &git.CommitOptions{
|
||||
Parents: []plumbing.Hash{headRef.Hash()},
|
||||
Author: &object.Signature{
|
||||
Name: "gitea-actions",
|
||||
Email: "info@unistack.org",
|
||||
When: time.Now(),
|
||||
},
|
||||
}) // хотим за коммитить изменения
|
||||
if err != nil {
|
||||
g.logger.Fatal(ctx, fmt.Sprintf("failed to commit: %v", err))
|
||||
}
|
||||
|
||||
refspec := gitconfig.RefSpec(fmt.Sprintf("+refs/heads/pkgdash/go_modules/%s-%s:refs/heads/pkgdash/go_modules/%s-%s", path, mod.Version, path, mod.Version)) // todo как будто нужно переделать
|
||||
|
||||
g.logger.Info(ctx, fmt.Sprintf("try to push refspec %s", refspec))
|
||||
|
||||
if err = repo.PushContext(ctx, &git.PushOptions{
|
||||
RefSpecs: []gitconfig.RefSpec{refspec},
|
||||
Auth: &httpauth.BasicAuth{Username: g.Username, Password: g.Password},
|
||||
Force: true,
|
||||
}); err != nil {
|
||||
g.logger.Fatal(ctx, fmt.Sprintf("failed to push repo branch: %v", err))
|
||||
} // пытаемся за пушить изменения
|
||||
|
||||
body := map[string]string{
|
||||
"base": branch,
|
||||
"body": wBody.String(),
|
||||
"head": fmt.Sprintf("pkgdash/go_modules/%s-%s", path, mod.Version),
|
||||
"title": wTitle.String(),
|
||||
}
|
||||
g.logger.Info(ctx, fmt.Sprintf("raw body: %#+v", body))
|
||||
|
||||
buf, err = json.Marshal(body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
g.logger.Info(ctx, fmt.Sprintf("marshal body: %s", buf))
|
||||
|
||||
req, err := http.NewRequestWithContext(
|
||||
ctx,
|
||||
http.MethodPost,
|
||||
fmt.Sprintf("https://%s/api/v1/repos/%s/%s/pulls", g.URL, g.Owner, g.Repository),
|
||||
bytes.NewReader(buf),
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req.Header.Add("Accept", "application/json")
|
||||
req.Header.Add("Content-Type", "application/json")
|
||||
req.Header.Add("Authorization", "Bearer "+g.Password)
|
||||
|
||||
rsp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
} // Вроде создаем новый реквест на создание пулл реквеста
|
||||
if rsp.StatusCode != http.StatusCreated {
|
||||
buf, _ = io.ReadAll(rsp.Body)
|
||||
return fmt.Errorf("unknown error: %s", buf)
|
||||
}
|
||||
|
||||
g.logger.Info(ctx, fmt.Sprintf("PR create for %s-%s", path, mod.Version))
|
||||
|
||||
repo, err = git.PlainOpenWithOptions(".", &git.PlainOpenOptions{DetectDotGit: true})
|
||||
if err != nil {
|
||||
g.logger.Fatal(ctx, fmt.Sprintf("failed to open repo: %v", err))
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (g *Github) RequestClose(ctx context.Context, branch string, path string) error {
|
||||
return fmt.Errorf("implement me")
|
||||
}
|
||||
|
||||
func (g *Github) RequestUpdate(ctx context.Context, branch string, path string, mod modules.Update) error {
|
||||
return fmt.Errorf("implement me")
|
||||
}
|
||||
|
||||
func (g *Github) RequestList(ctx context.Context, branch string) (map[string]string, error) {
|
||||
g.logger.Debug(ctx, fmt.Sprintf("RequestList for %s", branch))
|
||||
var err error
|
||||
|
||||
g.pulls, err = g.GetPulls(ctx, g.URL, g.Owner, g.Repository, g.Password)
|
||||
if err != nil {
|
||||
g.logger.Error(ctx, fmt.Sprintf("GetPulls error: %s", err))
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var path string
|
||||
rMap := make(map[string]string)
|
||||
|
||||
for _, pull := range g.pulls {
|
||||
if !strings.HasPrefix(pull.Title, "Bump ") || pull.Base.Ref != branch { // добавляем только реквесты бота по обновлению модулей
|
||||
continue
|
||||
}
|
||||
path = strings.Split(pull.Title, " ")[1] // todo Работет только для дефолтного шаблона
|
||||
rMap[path] = pull.Title
|
||||
}
|
||||
return rMap, nil
|
||||
}
|
||||
|
||||
func (g *Github) GetPulls(ctx context.Context, url, owner, repo, password string) ([]*githubPull, error) {
|
||||
var pullsAll []*githubPull
|
||||
page := 1
|
||||
|
||||
for {
|
||||
pulls := make([]*githubPull, 0, 10)
|
||||
req, err := http.NewRequestWithContext(
|
||||
ctx,
|
||||
http.MethodGet,
|
||||
fmt.Sprintf("https://%s/api/v1/repos/%s/%s/pulls?state=open&page=%v", url, owner, repo, page),
|
||||
nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
} // вроде запроса к репозиторию
|
||||
|
||||
req.Header.Add("Accept", "application/json")
|
||||
req.Header.Add("Content-Type", "application/json")
|
||||
req.Header.Add("Authorization", "Bearer "+password)
|
||||
|
||||
rsp, err := http.DefaultClient.Do(req) // выполнение запроса
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
buf, _ := io.ReadAll(rsp.Body)
|
||||
|
||||
switch rsp.StatusCode {
|
||||
case http.StatusOK:
|
||||
if err = json.Unmarshal(buf, &pulls); err != nil {
|
||||
g.logger.Error(ctx, fmt.Sprintf("failed to decode response %s err: %v", buf, err))
|
||||
return nil, err
|
||||
}
|
||||
pullsAll = append(pullsAll, pulls...)
|
||||
page++
|
||||
case http.StatusNotFound:
|
||||
g.logger.Info(ctx, fmt.Sprintf("pull-request is not exist for %s", repo))
|
||||
return nil, ErrPRNotExist
|
||||
default:
|
||||
return nil, fmt.Errorf("unknown error: %s", buf)
|
||||
}
|
||||
|
||||
if len(pulls) == 0 {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return pullsAll, nil
|
||||
}
|
||||
|
||||
func (g *Github) checkout(w git.Worktree, ref plumbing.Reference) {
|
||||
ctx := context.Background()
|
||||
g.logger.Debug(ctx, fmt.Sprintf("Checkout: %s", ref.Name().Short()))
|
||||
|
||||
if err := w.Checkout(&git.CheckoutOptions{
|
||||
Branch: ref.Name(),
|
||||
Create: false,
|
||||
Force: true,
|
||||
Keep: false,
|
||||
}); err != nil {
|
||||
g.logger.Error(ctx, fmt.Sprintf("failed to reset: %v", err))
|
||||
}
|
||||
}
|
||||
522
internal/source/gitlab/gitlab.go
Normal file
522
internal/source/gitlab/gitlab.go
Normal file
@@ -0,0 +1,522 @@
|
||||
package gitlab
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"os/exec"
|
||||
"regexp"
|
||||
"strconv"
|
||||
"strings"
|
||||
"text/template"
|
||||
"time"
|
||||
|
||||
"github.com/go-git/go-git/v5"
|
||||
gitconfig "github.com/go-git/go-git/v5/config"
|
||||
"github.com/go-git/go-git/v5/plumbing"
|
||||
"github.com/go-git/go-git/v5/plumbing/object"
|
||||
httpauth "github.com/go-git/go-git/v5/plumbing/transport/http"
|
||||
"go.unistack.org/micro/v3/logger"
|
||||
"go.unistack.org/pkgdash/internal/configcli"
|
||||
"go.unistack.org/pkgdash/internal/modules"
|
||||
)
|
||||
|
||||
var ErrPRNotExist = errors.New("pull request does not exist")
|
||||
|
||||
type Gitlab struct {
|
||||
logger logger.Logger
|
||||
URL string
|
||||
Username string
|
||||
Password string
|
||||
PRTitle string
|
||||
PRBody string
|
||||
Repository string
|
||||
RepositoryId string
|
||||
Owner string
|
||||
pulls []*gitlabPull
|
||||
baseRef *plumbing.Reference
|
||||
}
|
||||
|
||||
func NewGitlab(cfg configcli.Config, log logger.Logger) *Gitlab {
|
||||
return &Gitlab{
|
||||
logger: log,
|
||||
URL: cfg.Source.APIURL,
|
||||
Username: cfg.Source.Username,
|
||||
Password: cfg.Source.Password,
|
||||
PRTitle: cfg.PullRequestTitle,
|
||||
PRBody: cfg.PullRequestBody,
|
||||
Repository: cfg.Source.Repository,
|
||||
Owner: cfg.Source.Owner,
|
||||
}
|
||||
}
|
||||
|
||||
type gitlabPull struct {
|
||||
URL string `json:"web_url"`
|
||||
Title string `json:"title"`
|
||||
Target string `json:"target_branch"`
|
||||
Source string `json:"source_branch"`
|
||||
ID int64 `json:"id"`
|
||||
}
|
||||
|
||||
type gitlabProject struct {
|
||||
Id int64 `json:"id"`
|
||||
Name string `json:"name"`
|
||||
}
|
||||
|
||||
func (g *Gitlab) Name() string {
|
||||
return "gitlab"
|
||||
}
|
||||
|
||||
func (g *Gitlab) RequestOpen(ctx context.Context, branch string, path string, mod modules.Update) error {
|
||||
g.logger.Debug(ctx, fmt.Sprintf("RequestOpen start, mod title: %s", path))
|
||||
|
||||
var buf []byte
|
||||
var err error
|
||||
// создания шаблона названия для пулл реквеста
|
||||
tplTitle, err := template.New("pull_request_title").Parse(g.PRTitle)
|
||||
if err != nil {
|
||||
g.logger.Fatal(ctx, fmt.Sprintf("failed to parse template: %v", err))
|
||||
}
|
||||
|
||||
wTitle := bytes.NewBuffer(nil)
|
||||
// создания шаблона тела для пулл реквеста
|
||||
tplBody, err := template.New("pull_request_body").Parse(g.PRTitle)
|
||||
if err != nil {
|
||||
g.logger.Fatal(ctx, fmt.Sprintf("failed to parse template: %v", err))
|
||||
}
|
||||
|
||||
wBody := bytes.NewBuffer(nil)
|
||||
|
||||
data := map[string]string{
|
||||
"Name": path,
|
||||
"VersionOld": mod.Module.Version,
|
||||
"VersionNew": mod.Version,
|
||||
}
|
||||
|
||||
if err = tplTitle.Execute(wTitle, data); err != nil {
|
||||
g.logger.Fatal(ctx, fmt.Sprintf("failed to execute template: %v", err))
|
||||
}
|
||||
if err = tplBody.Execute(wBody, data); err != nil {
|
||||
g.logger.Fatal(ctx, fmt.Sprintf("failed to execute template: %v", err))
|
||||
}
|
||||
|
||||
// открытие гит репозитория с опцией обхода репозитория для нахождения .git
|
||||
repo, err := git.PlainOpenWithOptions(".", &git.PlainOpenOptions{DetectDotGit: true})
|
||||
if err != nil {
|
||||
g.logger.Fatal(ctx, fmt.Sprintf("failed to open repo: %v", err))
|
||||
}
|
||||
// извлекаем ссылки с объектами из удаленного объекта??
|
||||
if err = repo.FetchContext(ctx, &git.FetchOptions{
|
||||
// Auth: &httpauth.BasicAuth{Username: g.Username, Password: g.Password},
|
||||
Force: true,
|
||||
}); err != nil && err != git.NoErrAlreadyUpToDate {
|
||||
g.logger.Fatal(ctx, fmt.Sprintf("failed to fetch repo : %v", err))
|
||||
} // обновляем репозиторий
|
||||
|
||||
var headRef *plumbing.Reference // вроде ссылка на гит
|
||||
|
||||
if g.baseRef == nil {
|
||||
g.baseRef, err = repo.Head()
|
||||
if err != nil {
|
||||
g.logger.Fatal(ctx, fmt.Sprintf("Error head: %s", err))
|
||||
}
|
||||
}
|
||||
|
||||
refIter, err := repo.Branches() // получение веток
|
||||
if err != nil {
|
||||
g.logger.Fatal(ctx, fmt.Sprintf("failed to get branches: %v", err))
|
||||
}
|
||||
for {
|
||||
ref, err := refIter.Next()
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
if ref.Name().Short() == branch { // todo вот тут возможно нужно переделать
|
||||
headRef = ref
|
||||
break
|
||||
}
|
||||
} // перебираем получение ветки и когда находим нужную выходим из цикла записав ветку в headRef
|
||||
refIter.Close()
|
||||
|
||||
if headRef == nil {
|
||||
g.logger.Fatal(ctx, "failed to get repo branch head")
|
||||
return err
|
||||
} // Не получили нужную ветку
|
||||
|
||||
g.logger.Info(ctx, fmt.Sprintf("repo head %s", headRef))
|
||||
|
||||
wtree, err := repo.Worktree() // todo вроде рабочее дерево не нужно
|
||||
if err != nil {
|
||||
g.logger.Fatal(ctx, fmt.Sprintf("failed to get worktree: %v", err))
|
||||
}
|
||||
defer g.checkout(*wtree, *g.baseRef)
|
||||
|
||||
g.pulls, err = g.GetPulls(ctx, g.URL, g.RepositoryId, branch, g.Password)
|
||||
if err != nil && err != ErrPRNotExist {
|
||||
g.logger.Error(ctx, fmt.Sprintf("GetPulls error: %s", err))
|
||||
return err
|
||||
}
|
||||
|
||||
for _, pull := range g.pulls {
|
||||
if strings.Contains(pull.Title, path) {
|
||||
g.logger.Info(ctx, fmt.Sprintf("PR for %s exists %s, call RequestUpdate", path, pull.URL))
|
||||
return g.RequestUpdate(ctx, branch, path, mod)
|
||||
} // хотим проверить есть ли пулл реквест для этой ветки, если есть то выходим
|
||||
}
|
||||
|
||||
g.logger.Info(ctx, fmt.Sprintf("update %s from %s to %s", path, mod.Module.Version, mod.Version))
|
||||
|
||||
sourceBranch := fmt.Sprintf("pkgdash/go_modules/%s-%s", path, mod.Version)
|
||||
|
||||
g.logger.Info(ctx, "reset worktree")
|
||||
if err = wtree.Reset(&git.ResetOptions{Commit: headRef.Hash(), Mode: git.HardReset}); err != nil {
|
||||
g.logger.Error(ctx, fmt.Sprintf("failed to reset repo branch: %v", err))
|
||||
}
|
||||
|
||||
if err = wtree.PullContext(ctx, &git.PullOptions{
|
||||
Auth: &httpauth.BasicAuth{Username: g.Username, Password: g.Password},
|
||||
Depth: 1,
|
||||
// RemoteURL :
|
||||
Force: true,
|
||||
RemoteName: "origin",
|
||||
}); err != nil && err != git.NoErrAlreadyUpToDate {
|
||||
g.logger.Error(ctx, fmt.Sprintf("failed to pull repo: %v", err)) // подтягиваем изменения с удаленого репозитория
|
||||
}
|
||||
|
||||
g.logger.Info(ctx, fmt.Sprintf("checkout ref %s", headRef))
|
||||
if err = wtree.Checkout(&git.CheckoutOptions{
|
||||
Hash: headRef.Hash(),
|
||||
Branch: plumbing.NewBranchReferenceName(sourceBranch),
|
||||
Create: true,
|
||||
Force: true,
|
||||
}); err != nil {
|
||||
g.logger.Error(ctx, fmt.Sprintf("failed to checkout tree: %v", err))
|
||||
return err
|
||||
} // создаем новую ветку
|
||||
|
||||
epath, err := exec.LookPath("go")
|
||||
if errors.Is(err, exec.ErrDot) {
|
||||
err = nil
|
||||
}
|
||||
if err != nil {
|
||||
g.logger.Fatal(ctx, fmt.Sprintf("failed to find go command: %v", err))
|
||||
} // ищем go файл
|
||||
|
||||
var cmd *exec.Cmd
|
||||
var out []byte
|
||||
|
||||
cmd = exec.CommandContext(ctx, epath, "mod", "edit", fmt.Sprintf("-droprequire=%s", mod.Module.Path))
|
||||
if out, err = cmd.CombinedOutput(); err != nil {
|
||||
g.logger.Fatal(ctx, fmt.Sprintf("failed to run go mod edit: %s err: %v", out, err))
|
||||
}
|
||||
|
||||
cmd = exec.CommandContext(ctx, epath, "mod", "edit", fmt.Sprintf("-require=%s@%s", path, mod.Version))
|
||||
if out, err = cmd.CombinedOutput(); err != nil {
|
||||
g.logger.Fatal(ctx, fmt.Sprintf("failed to run go mod edit: %s err: %v", out, err))
|
||||
} // пытаемся выполнить команду go mod edit с новой версией модуля
|
||||
|
||||
cmd = exec.CommandContext(ctx, epath, "mod", "tidy")
|
||||
if out, err = cmd.CombinedOutput(); err != nil {
|
||||
g.logger.Fatal(ctx, fmt.Sprintf("failed to run go mod tidy: %s err: %v", out, err))
|
||||
} // пытаемся выполнить команду go mod tidy пытаемся подтянуть новую версию модуля
|
||||
|
||||
g.logger.Info(ctx, "worktree add go.mod")
|
||||
if _, err = wtree.Add("go.mod"); err != nil {
|
||||
g.logger.Fatal(ctx, fmt.Sprintf("failed to add file: %v", err))
|
||||
}
|
||||
|
||||
g.logger.Info(ctx, "worktree add go.sum")
|
||||
if _, err = wtree.Add("go.sum"); err != nil {
|
||||
g.logger.Fatal(ctx, fmt.Sprintf("failed to add file: %v", err))
|
||||
}
|
||||
|
||||
g.logger.Info(ctx, "worktree commit")
|
||||
_, err = wtree.Commit(wTitle.String(), &git.CommitOptions{
|
||||
Parents: []plumbing.Hash{headRef.Hash()},
|
||||
Author: &object.Signature{
|
||||
Name: "gitea-actions",
|
||||
Email: "info@unistack.org",
|
||||
When: time.Now(),
|
||||
},
|
||||
}) // хотим за коммитить изменения
|
||||
if err != nil {
|
||||
g.logger.Fatal(ctx, fmt.Sprintf("failed to commit: %v", err))
|
||||
}
|
||||
|
||||
refspec := gitconfig.RefSpec(fmt.Sprintf("+refs/heads/pkgdash/go_modules/%s-%s:refs/heads/pkgdash/go_modules/%s-%s", path, mod.Version, path, mod.Version)) // todo как будто нужно переделать
|
||||
|
||||
g.logger.Info(ctx, fmt.Sprintf("try to push refspec %s", refspec))
|
||||
|
||||
if err = repo.PushContext(ctx, &git.PushOptions{
|
||||
RefSpecs: []gitconfig.RefSpec{refspec},
|
||||
Auth: &httpauth.BasicAuth{Username: g.Username, Password: g.Password},
|
||||
Force: true,
|
||||
}); err != nil {
|
||||
g.logger.Fatal(ctx, fmt.Sprintf("failed to push repo branch: %v", err))
|
||||
} // пытаемся за пушить изменения
|
||||
|
||||
body := map[string]string{
|
||||
"id": g.RepositoryId,
|
||||
"source_branch": sourceBranch,
|
||||
"target_branch": branch,
|
||||
"title": wTitle.String(),
|
||||
"description": wBody.String(),
|
||||
}
|
||||
g.logger.Info(ctx, fmt.Sprintf("raw body: %#+v", body))
|
||||
|
||||
buf, err = json.Marshal(body)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
g.logger.Info(ctx, fmt.Sprintf("marshal body: %s", buf))
|
||||
|
||||
req, err := http.NewRequestWithContext(
|
||||
ctx,
|
||||
http.MethodPost,
|
||||
fmt.Sprintf("https://%s/api/v4/projects/%s/merge_requests", g.URL, g.RepositoryId),
|
||||
bytes.NewReader(buf),
|
||||
)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
req.Header.Add("Accept", "application/json")
|
||||
req.Header.Add("Content-Type", "application/json")
|
||||
req.Header.Add("Authorization", "Bearer "+g.Password)
|
||||
|
||||
rsp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
return err
|
||||
} // Вроде создаем новый реквест на создание пулл реквеста
|
||||
if rsp.StatusCode != http.StatusCreated {
|
||||
buf, _ = io.ReadAll(rsp.Body)
|
||||
return fmt.Errorf("unknown error: %s", buf)
|
||||
}
|
||||
|
||||
g.logger.Info(ctx, fmt.Sprintf("PR create for %s-%s", path, mod.Version))
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (g *Gitlab) RequestClose(ctx context.Context, branch string, path string) error {
|
||||
g.logger.Debug(ctx, fmt.Sprintf("RequestClose start, mod title: %s", path))
|
||||
var err error
|
||||
|
||||
g.pulls, err = g.GetPulls(ctx, g.URL, g.RepositoryId, branch, g.Password)
|
||||
if err != nil {
|
||||
g.logger.Error(ctx, fmt.Sprintf("GetPulls error: %s", err))
|
||||
return err
|
||||
}
|
||||
|
||||
prExist := false
|
||||
var b string // Name of the branch to be deleted
|
||||
for _, pull := range g.pulls {
|
||||
if strings.Contains(pull.Title, path) {
|
||||
g.logger.Info(ctx, fmt.Sprintf("PR for %s exists: %s", path, pull.URL))
|
||||
prExist = true
|
||||
b = pull.Source
|
||||
}
|
||||
}
|
||||
if !prExist {
|
||||
g.logger.Error(ctx, fmt.Sprintf("skip %s since pr does not exist", path))
|
||||
return ErrPRNotExist
|
||||
}
|
||||
|
||||
req, err := g.DeleteBranch(ctx, g.URL, g.RepositoryId, b, g.Password)
|
||||
if err != nil {
|
||||
g.logger.Error(ctx, fmt.Sprintf("failed to create request for delete the branch: %s, err: %s", branch, err))
|
||||
return err
|
||||
}
|
||||
rsp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
g.logger.Error(ctx, fmt.Sprintf("failed to do request for delete the branch: %s, err: %s, code: %v", branch, err, rsp.StatusCode))
|
||||
return err
|
||||
}
|
||||
|
||||
g.logger.Info(ctx, fmt.Sprintf("Delete branch for %s successful", path))
|
||||
return nil
|
||||
}
|
||||
|
||||
func (g *Gitlab) RequestUpdate(ctx context.Context, branch string, path string, mod modules.Update) error {
|
||||
g.logger.Debug(ctx, fmt.Sprintf("RequestUpdate start, mod title: %s", path))
|
||||
var err error
|
||||
|
||||
g.RepositoryId, err = g.GetRepoID(ctx, g.URL, g.Owner, g.Repository, g.Password)
|
||||
if err != nil || g.RepositoryId == "" {
|
||||
return fmt.Errorf("project id is empty")
|
||||
}
|
||||
|
||||
g.pulls, err = g.GetPulls(ctx, g.URL, g.RepositoryId, branch, g.Password)
|
||||
if err != nil {
|
||||
g.logger.Error(ctx, fmt.Sprintf("GetPulls error: %s", err))
|
||||
return err
|
||||
}
|
||||
|
||||
prExist := false
|
||||
for _, pull := range g.pulls {
|
||||
if strings.Contains(pull.Title, path) {
|
||||
g.logger.Info(ctx, fmt.Sprintf("don't skip %s since pr exist %s", path, pull.URL)) // todo
|
||||
tVersion := getVersions(pull.Source) // Надо взять просто из названия ветки последнюю версию
|
||||
if modules.IsNewerVersion(tVersion, mod.Version, false) {
|
||||
reqDel, err := g.DeleteBranch(ctx, g.URL, g.RepositoryId, pull.Source, g.Password)
|
||||
if err != nil {
|
||||
g.logger.Error(ctx, fmt.Sprintf("Error with create request for branch: %s, err: %s", branch, err))
|
||||
return err
|
||||
}
|
||||
rsp, err := http.DefaultClient.Do(reqDel)
|
||||
if err != nil {
|
||||
g.logger.Error(ctx, fmt.Sprintf("Error with do request for branch: %s, err: %s, code: %v", branch, err, rsp.StatusCode))
|
||||
return err
|
||||
}
|
||||
g.logger.Info(ctx, fmt.Sprintf("Old pr %s successful delete", pull.Source))
|
||||
} else {
|
||||
g.logger.Debug(ctx, "The existing PR is relevant")
|
||||
return nil
|
||||
}
|
||||
prExist = true
|
||||
}
|
||||
}
|
||||
if !prExist {
|
||||
g.logger.Error(ctx, fmt.Sprintf("skip %s since pr does not exist", path))
|
||||
return ErrPRNotExist
|
||||
}
|
||||
|
||||
return g.RequestOpen(ctx, branch, path, mod) // todo это мне не нравится
|
||||
}
|
||||
|
||||
func (g *Gitlab) RequestList(ctx context.Context, branch string) (map[string]string, error) {
|
||||
g.logger.Debug(ctx, fmt.Sprintf("RequestList for %s", branch))
|
||||
var err error
|
||||
|
||||
g.RepositoryId, err = g.GetRepoID(ctx, g.URL, g.Owner, g.Repository, g.Password)
|
||||
if err != nil || g.RepositoryId == "" {
|
||||
return nil, fmt.Errorf("project id is empty")
|
||||
}
|
||||
|
||||
g.pulls, err = g.GetPulls(ctx, g.URL, g.RepositoryId, branch, g.Password)
|
||||
if err != nil {
|
||||
g.logger.Error(ctx, fmt.Sprintf("GetPulls error: %s", err))
|
||||
return nil, err
|
||||
}
|
||||
|
||||
var path string
|
||||
rMap := make(map[string]string)
|
||||
|
||||
for _, pull := range g.pulls {
|
||||
if !strings.HasPrefix(pull.Title, "Bump ") { // добавляем только реквесты бота по обновлению модулей
|
||||
continue
|
||||
}
|
||||
path = strings.Split(pull.Title, " ")[1] // todo Работет только для дефолтного шаблона
|
||||
rMap[path] = pull.Title
|
||||
}
|
||||
return rMap, nil
|
||||
}
|
||||
|
||||
func getVersions(s string) string {
|
||||
re := regexp.MustCompile("[vV][0-9]+\\.[0-9]+\\.[0-9]+")
|
||||
|
||||
version := re.FindString(s)
|
||||
|
||||
return version
|
||||
}
|
||||
|
||||
func (g *Gitlab) DeleteBranch(ctx context.Context, url, projectId, branch, password string) (*http.Request, error) {
|
||||
var buf []byte
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodDelete, fmt.Sprintf("https://%s/api/v4/projects/%s/repository/branches/%s", url, projectId, branch), bytes.NewReader(buf))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
req.Header.Add("Accept", "application/json")
|
||||
req.Header.Add("Content-Type", "application/json")
|
||||
req.Header.Add("Authorization", "Bearer "+password)
|
||||
return req, err
|
||||
}
|
||||
|
||||
func (g *Gitlab) GetPulls(ctx context.Context, url, projectId, branch, password string) ([]*gitlabPull, error) {
|
||||
pulls := make([]*gitlabPull, 0, 10)
|
||||
req, err := http.NewRequestWithContext(
|
||||
ctx,
|
||||
http.MethodGet,
|
||||
fmt.Sprintf("https://%s/api/v4/projects/%s/merge_requests?state=opened&target_branch=%s", url, projectId, branch),
|
||||
nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
} // вроде запроса к репозиторию
|
||||
|
||||
req.Header.Add("Accept", "application/json")
|
||||
req.Header.Add("Content-Type", "application/json")
|
||||
req.Header.Add("Authorization", "Bearer "+password)
|
||||
|
||||
rsp, err := http.DefaultClient.Do(req) // выполнение запроса
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
buf, _ := io.ReadAll(rsp.Body)
|
||||
|
||||
switch rsp.StatusCode {
|
||||
case http.StatusOK:
|
||||
if err = json.Unmarshal(buf, &pulls); err != nil {
|
||||
g.logger.Error(ctx, fmt.Sprintf("failed to decode response %s err: %v", buf, err))
|
||||
return nil, err
|
||||
}
|
||||
return pulls, nil
|
||||
case http.StatusNotFound:
|
||||
g.logger.Info(ctx, fmt.Sprintf("pull-request is not exist for %s", projectId))
|
||||
return nil, ErrPRNotExist
|
||||
default:
|
||||
return nil, fmt.Errorf("unknown error: %s", buf)
|
||||
}
|
||||
}
|
||||
|
||||
func (g *Gitlab) GetRepoID(ctx context.Context, url, owner, repo, password string) (rId string, err error) {
|
||||
var buf []byte
|
||||
projects := make([]*gitlabProject, 0, 10)
|
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, fmt.Sprintf("https://%s/api/v4/users/%s/projects?owned=true", url, owner), nil)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
req.Header.Add("Accept", "application/json")
|
||||
req.Header.Add("Content-Type", "application/json")
|
||||
req.Header.Add("Authorization", password)
|
||||
|
||||
rsp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
buf, _ = io.ReadAll(rsp.Body)
|
||||
|
||||
switch rsp.StatusCode {
|
||||
case http.StatusOK:
|
||||
if err = json.Unmarshal(buf, &projects); err != nil {
|
||||
g.logger.Error(ctx, fmt.Sprintf("failed to decode response %s err: %v", buf, err))
|
||||
}
|
||||
for _, p := range projects {
|
||||
if p.Name == repo {
|
||||
rId = strconv.Itoa(int(p.Id))
|
||||
}
|
||||
}
|
||||
return
|
||||
default:
|
||||
return rId, fmt.Errorf("unknown error: %s", buf)
|
||||
}
|
||||
}
|
||||
|
||||
func (g *Gitlab) checkout(w git.Worktree, ref plumbing.Reference) {
|
||||
ctx := context.Background()
|
||||
g.logger.Debug(ctx, fmt.Sprintf("Checkout: %s", ref.Name().Short()))
|
||||
|
||||
if err := w.Checkout(&git.CheckoutOptions{
|
||||
Branch: ref.Name(),
|
||||
Create: false,
|
||||
Force: true,
|
||||
Keep: false,
|
||||
}); err != nil {
|
||||
g.logger.Error(ctx, fmt.Sprintf("failed to reset: %v", err))
|
||||
}
|
||||
}
|
||||
44
internal/source/gogs/gogs.go
Normal file
44
internal/source/gogs/gogs.go
Normal file
@@ -0,0 +1,44 @@
|
||||
package gogs
|
||||
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
|
||||
"go.unistack.org/micro/v3/logger"
|
||||
"go.unistack.org/pkgdash/internal/configcli"
|
||||
"go.unistack.org/pkgdash/internal/modules"
|
||||
)
|
||||
|
||||
type Gogs struct {
|
||||
logger logger.Logger
|
||||
Username string
|
||||
Password string
|
||||
}
|
||||
|
||||
func NewGogs(cfg configcli.Config, log logger.Logger) *Gogs {
|
||||
return &Gogs{
|
||||
logger: log,
|
||||
Username: cfg.Source.Username,
|
||||
Password: cfg.Source.Password,
|
||||
}
|
||||
}
|
||||
|
||||
func (g *Gogs) Name() string {
|
||||
return "gogs"
|
||||
}
|
||||
|
||||
func (g *Gogs) RequestOpen(ctx context.Context, branch string, path string, mod modules.Update) error {
|
||||
return fmt.Errorf("implement me")
|
||||
}
|
||||
|
||||
func (g *Gogs) RequestClose(ctx context.Context, branch string, path string) error {
|
||||
return fmt.Errorf("implement me")
|
||||
}
|
||||
|
||||
func (g *Gogs) RequestUpdate(ctx context.Context, branch string, path string, mod modules.Update) error {
|
||||
return fmt.Errorf("implement me")
|
||||
}
|
||||
|
||||
func (g *Gogs) RequestList(ctx context.Context, branch string) (map[string]string, error) {
|
||||
return nil, fmt.Errorf("implement me")
|
||||
}
|
||||
35
internal/source/source.go
Normal file
35
internal/source/source.go
Normal file
@@ -0,0 +1,35 @@
|
||||
package source
|
||||
|
||||
import (
|
||||
"context"
|
||||
|
||||
"go.unistack.org/micro/v3/logger"
|
||||
"go.unistack.org/pkgdash/internal/configcli"
|
||||
"go.unistack.org/pkgdash/internal/modules"
|
||||
"go.unistack.org/pkgdash/internal/source/gitea"
|
||||
"go.unistack.org/pkgdash/internal/source/github"
|
||||
"go.unistack.org/pkgdash/internal/source/gitlab"
|
||||
"go.unistack.org/pkgdash/internal/source/gogs"
|
||||
)
|
||||
|
||||
type SourceControl interface {
|
||||
Name() string
|
||||
RequestOpen(ctx context.Context, branch string, path string, mod modules.Update) error
|
||||
RequestClose(ctx context.Context, branch string, path string) error
|
||||
RequestUpdate(ctx context.Context, branch string, path string, mod modules.Update) error
|
||||
RequestList(ctx context.Context, branch string) (map[string]string, error)
|
||||
}
|
||||
|
||||
func NewSourceControl(cfg configcli.Config, log logger.Logger) SourceControl {
|
||||
switch cfg.Source.TypeGit {
|
||||
case "github":
|
||||
return github.NewGithub(cfg, log)
|
||||
case "gitlab":
|
||||
return gitlab.NewGitlab(cfg, log)
|
||||
case "gitea":
|
||||
return gitea.NewGitea(cfg, log)
|
||||
case "gogs":
|
||||
return gogs.NewGogs(cfg, log)
|
||||
}
|
||||
return nil
|
||||
}
|
||||
34
internal/storage/postgres_tmp/queries.go
Normal file
34
internal/storage/postgres_tmp/queries.go
Normal file
@@ -0,0 +1,34 @@
|
||||
//go:build ignore
|
||||
|
||||
package postgres
|
||||
|
||||
const (
|
||||
queryListPackage = `
|
||||
select
|
||||
id,
|
||||
name,
|
||||
url,
|
||||
comments
|
||||
--modules,
|
||||
--issues,
|
||||
from package;
|
||||
`
|
||||
queryAddComment = `
|
||||
with insert_comm as (
|
||||
insert into comment(text) values ($1) returning id
|
||||
)
|
||||
update package set comments = array_append(comments, (select * from insert_comm)) where id=$2;
|
||||
`
|
||||
queryAddPackage = `
|
||||
insert into package(name, url, modules) values ($1, $2, $3);
|
||||
`
|
||||
queryInsMsgGetIDs = `
|
||||
insert into module(name, version, last_version) values
|
||||
%s
|
||||
returning id;
|
||||
`
|
||||
queryGetModule = `
|
||||
select id, name, version, last_version from module
|
||||
where id in %s ;
|
||||
`
|
||||
)
|
||||
235
internal/storage/postgres_tmp/storage.go
Normal file
235
internal/storage/postgres_tmp/storage.go
Normal file
@@ -0,0 +1,235 @@
|
||||
//go:build ignore
|
||||
|
||||
package postgres
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"embed"
|
||||
"errors"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/golang-migrate/migrate/v3"
|
||||
mpgx "github.com/golang-migrate/migrate/v4/database/pgx"
|
||||
"github.com/golang-migrate/migrate/v4/source/iofs"
|
||||
"github.com/lib/pq"
|
||||
"go.unistack.org/micro/v3/logger"
|
||||
"go.unistack.org/pkgdash/internal/config"
|
||||
"go.unistack.org/pkgdash/internal/models"
|
||||
pb "go.unistack.org/pkgdash/proto"
|
||||
)
|
||||
|
||||
const (
|
||||
pathMigration = `migrations/postgres`
|
||||
)
|
||||
|
||||
type Postgres struct {
|
||||
db *sql.DB
|
||||
fs embed.FS
|
||||
}
|
||||
|
||||
func NewStorage() func(*sql.DB, embed.FS) interface{} {
|
||||
return func(db *sql.DB, fs embed.FS) interface{} {
|
||||
return &Postgres{db: db, fs: fs}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *Postgres) MigrateUp() error {
|
||||
driver, err := mpgx.WithInstance(s.db, &mpgx.Config{
|
||||
MigrationsTable: mpgx.DefaultMigrationsTable,
|
||||
DatabaseName: config.ServiceName,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
source, err := iofs.New(s.fs, pathMigration)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// TODO: pass own logger
|
||||
m, err := migrate.NewWithInstance("fs", source, config.ServiceName, driver)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err = m.Up(); err != nil && !errors.Is(err, migrate.ErrNoChange) {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Postgres) MigrateDown() error {
|
||||
driver, err := mpgx.WithInstance(s.db, &mpgx.Config{
|
||||
MigrationsTable: mpgx.DefaultMigrationsTable,
|
||||
DatabaseName: config.ServiceName,
|
||||
})
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
source, err := iofs.New(s.fs, pathMigration)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// TODO: pass own logger
|
||||
m, err := migrate.NewWithInstance("fs", source, config.ServiceName, driver)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if err = m.Down(); err != nil && !errors.Is(err, migrate.ErrNoChange) {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Postgres) PackagesUpdate(ctx context.Context, req *pb.PackagesUpdateReq) error {
|
||||
panic("need implement")
|
||||
}
|
||||
|
||||
func (s *Postgres) PackagesList(ctx context.Context, req *pb.PackagesListReq) (models.ListPackage, error) {
|
||||
rows, err := s.db.QueryContext(ctx, queryListPackage)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
defer func() {
|
||||
if err = rows.Close(); err != nil {
|
||||
return
|
||||
}
|
||||
err = rows.Err()
|
||||
}()
|
||||
|
||||
result := make([]*models.Package, 0)
|
||||
for rows.Next() {
|
||||
tmp := &models.Package{}
|
||||
if err = rows.Scan(
|
||||
&tmp.ID,
|
||||
&tmp.Name,
|
||||
&tmp.URL,
|
||||
pq.Array(&tmp.Comments),
|
||||
); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
}
|
||||
|
||||
return result, err
|
||||
}
|
||||
|
||||
func (s *Postgres) CommentsCreate(ctx context.Context, req *pb.CommentsCreateReq) error {
|
||||
tx, err := s.db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
defer func() {
|
||||
if err != nil {
|
||||
if rollbackErr := tx.Rollback(); rollbackErr != nil {
|
||||
logger.Error(ctx, "AddComment: unable to rollback: %v", rollbackErr)
|
||||
}
|
||||
} else {
|
||||
err = tx.Commit()
|
||||
}
|
||||
}()
|
||||
|
||||
res, err := tx.ExecContext(ctx, queryAddComment, req.Text, req.PackageId)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if aff, affErr := res.RowsAffected(); err != nil {
|
||||
err = affErr
|
||||
} else if aff == 0 {
|
||||
err = errors.New("rows affected is 0")
|
||||
}
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *Postgres) PackagesCreate(ctx context.Context, req *pb.PackagesCreateReq) error {
|
||||
tx, err := s.db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
defer func() {
|
||||
if err != nil {
|
||||
if rollbackErr := tx.Rollback(); rollbackErr != nil {
|
||||
logger.Error(ctx, "AddPackage: unable to rollback: %v", rollbackErr)
|
||||
}
|
||||
} else {
|
||||
err = tx.Commit()
|
||||
}
|
||||
}()
|
||||
|
||||
res, err := tx.ExecContext(ctx, queryAddPackage, req.Name, req.Url, pq.Array(req.Modules))
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
if aff, affErr := res.RowsAffected(); err != nil {
|
||||
err = affErr
|
||||
} else if aff == 0 {
|
||||
err = errors.New("rows affected is 0")
|
||||
}
|
||||
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *Postgres) InsertButchModules(ctx context.Context, req []models.Module) ([]uint64, error) {
|
||||
tx, err := s.db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer func() {
|
||||
if err != nil {
|
||||
if rollbackErr := tx.Rollback(); rollbackErr != nil {
|
||||
logger.Error(ctx, "AddPackage: unable to rollback: %v", rollbackErr)
|
||||
}
|
||||
} else {
|
||||
err = tx.Commit()
|
||||
}
|
||||
}()
|
||||
|
||||
query := generateQuery(req)
|
||||
|
||||
rows, err := tx.QueryContext(ctx, query)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer func() {
|
||||
if err = rows.Close(); err != nil {
|
||||
return
|
||||
}
|
||||
err = rows.Err()
|
||||
}()
|
||||
|
||||
result := make([]uint64, 0)
|
||||
for rows.Next() {
|
||||
tmp := uint64(0)
|
||||
if err = rows.Scan(&tmp); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
result = append(result, tmp)
|
||||
}
|
||||
|
||||
return result, err
|
||||
}
|
||||
|
||||
func generateQuery(rsp []models.Module) string {
|
||||
const pattern = `%c('%s', '%s', '%s')`
|
||||
build := strings.Builder{}
|
||||
comma := ' '
|
||||
for i := range rsp {
|
||||
str := fmt.Sprintf(pattern, comma, rsp[i].Name, rsp[i].Version, rsp[i].LastVersion)
|
||||
build.WriteString(str)
|
||||
comma = ','
|
||||
}
|
||||
|
||||
return fmt.Sprintf(queryInsMsgGetIDs, build.String())
|
||||
}
|
||||
40
internal/storage/postgres_tmp/storage_test.go
Normal file
40
internal/storage/postgres_tmp/storage_test.go
Normal file
@@ -0,0 +1,40 @@
|
||||
//go:build ignore
|
||||
|
||||
package postgres
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"go.unistack.org/pkgdash/internal/models"
|
||||
)
|
||||
|
||||
func TestGenerate(t *testing.T) {
|
||||
m := []models.Module{
|
||||
{
|
||||
ID: 1,
|
||||
Name: "test",
|
||||
Version: "1.2.3",
|
||||
Package: 2,
|
||||
LastVersion: "23.31",
|
||||
},
|
||||
{
|
||||
ID: 1,
|
||||
Name: "321test",
|
||||
Version: "1.3",
|
||||
Package: 4,
|
||||
LastVersion: "2111.31",
|
||||
},
|
||||
{
|
||||
ID: 1,
|
||||
Name: "testabcd",
|
||||
Version: "1.2.3",
|
||||
Package: 2,
|
||||
LastVersion: "153453.31",
|
||||
},
|
||||
}
|
||||
|
||||
str := generateQuery(m)
|
||||
|
||||
fmt.Println(str)
|
||||
}
|
||||
19
internal/storage/sqlite/queries.go
Normal file
19
internal/storage/sqlite/queries.go
Normal file
@@ -0,0 +1,19 @@
|
||||
package sqlite
|
||||
|
||||
const (
|
||||
queryPackageModulesCount = `update packages set modules = $2 where id = $1;`
|
||||
queryPackagesModulesCreate = `insert into packages_modules as pm (package, module) values ($1, $2) on conflict (package,module) do nothing;`
|
||||
queryPackagesUpdateLastCheck = `update packages set last_check = CURRENT_TIMESTAMP where id = $1;`
|
||||
queryPackagesModules = `select modules.id, modules.name, modules.version from modules left join packages_modules on modules.id = packages_modules.module left join packages on packages.id = packages_modules.package where packages_modules.package = $1;`
|
||||
queryPackagesProcess = `select id, name, url, comments, modules, issues, created, updated, last_check from packages where ROUND((JULIANDAY(CURRENT_TIMESTAMP) - JULIANDAY(last_check)) * 86400) > $1 or last_check is NULL`
|
||||
queryModulesProcess = `select id, name, version, last_check from modules where ROUND((JULIANDAY(CURRENT_TIMESTAMP) - JULIANDAY(last_check)) * 86400) > $1 or last_check is NULL`
|
||||
queryPackagesModulesCount = `update packages set modules = $2, last_check = CURRENT_TIMESTAMP where id = $1;`
|
||||
queryPackagesList = `select id, name, url, comments, modules, issues, created, updated from packages;`
|
||||
queryPackagesLookup = `select id, name, url, comments, modules, issues, created, updated from packages where id = $1;`
|
||||
queryCommentsCreate = `insert into comments (comment) values ($1) returning id;`
|
||||
queryPackagesCreate = `insert into packages as p (name, url) values ($1, $2) on conflict (url) do update set name = p.name returning *;`
|
||||
queryModulesList = `select id, name, version from modules;`
|
||||
queryModulesCreate = `insert into modules as m (name, version) values ($1, $2) on conflict (name,version) do update set last_check = CURRENT_TIMESTAMP returning *;`
|
||||
queryCommentsList = `select id, text, created, updated from comments where package = $1;`
|
||||
queryHandlersList = `select id, name, coverage from handlers where package = $1;`
|
||||
)
|
||||
254
internal/storage/sqlite/storage.go
Normal file
254
internal/storage/sqlite/storage.go
Normal file
@@ -0,0 +1,254 @@
|
||||
package sqlite
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"time"
|
||||
|
||||
"github.com/jmoiron/sqlx"
|
||||
"go.unistack.org/micro/v3/logger"
|
||||
"go.unistack.org/pkgdash/internal/models"
|
||||
"go.unistack.org/pkgdash/internal/storage"
|
||||
pb "go.unistack.org/pkgdash/proto"
|
||||
)
|
||||
|
||||
func init() {
|
||||
storage.RegisterStorage("sqlite", NewStorage)
|
||||
}
|
||||
|
||||
var _ storage.Storage = (*Sqlite)(nil)
|
||||
|
||||
type Sqlite struct {
|
||||
logger logger.Logger
|
||||
db *sqlx.DB
|
||||
}
|
||||
|
||||
func NewStorage(log logger.Logger, db *sqlx.DB) interface{} {
|
||||
return &Sqlite{db: db, logger: log}
|
||||
}
|
||||
|
||||
func (s *Sqlite) PackageModulesCreate(ctx context.Context, pkg *models.Package, modules []*models.Module) error {
|
||||
tx, err := s.db.BeginTxx(ctx, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, mod := range modules {
|
||||
err = tx.GetContext(ctx, mod, queryModulesCreate, mod.Name, mod.Version)
|
||||
if err != nil {
|
||||
_ = tx.Rollback()
|
||||
return err
|
||||
}
|
||||
_, err = tx.ExecContext(ctx, queryPackagesModulesCreate, pkg.ID, mod.ID)
|
||||
if err != nil {
|
||||
_ = tx.Rollback()
|
||||
return err
|
||||
}
|
||||
|
||||
}
|
||||
|
||||
_, err = tx.ExecContext(ctx, queryPackageModulesCount, pkg.ID, len(modules))
|
||||
if err != nil {
|
||||
_ = tx.Rollback()
|
||||
return err
|
||||
}
|
||||
|
||||
if err = tx.Commit(); err != nil {
|
||||
_ = tx.Rollback()
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Sqlite) PackageDelete(ctx context.Context, req *pb.PackageDeleteReq) error {
|
||||
return fmt.Errorf("need implement")
|
||||
}
|
||||
|
||||
func (s *Sqlite) PackageUpdate(ctx context.Context, req *pb.PackageUpdateReq) (*models.Package, error) {
|
||||
return nil, fmt.Errorf("need implement")
|
||||
}
|
||||
|
||||
func (s *Sqlite) PackageLookup(ctx context.Context, req *pb.PackageLookupReq) (*models.Package, error) {
|
||||
pkg := &models.Package{}
|
||||
|
||||
err := s.db.GetContext(ctx, pkg, queryPackagesLookup, req.Id)
|
||||
if err != nil {
|
||||
if err == sql.ErrNoRows {
|
||||
return nil, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return pkg, err
|
||||
}
|
||||
|
||||
func (s *Sqlite) PackageList(ctx context.Context, req *pb.PackageListReq) ([]*models.Package, error) {
|
||||
var packages []*models.Package
|
||||
|
||||
err := s.db.SelectContext(ctx, &packages, queryPackagesList)
|
||||
if err != nil {
|
||||
if err == sql.ErrNoRows {
|
||||
return nil, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return packages, nil
|
||||
}
|
||||
|
||||
func (s *Sqlite) PackageModules(ctx context.Context, req *pb.PackageModulesReq) ([]*models.Module, error) {
|
||||
var modules []*models.Module
|
||||
|
||||
err := s.db.SelectContext(ctx, &modules, queryPackagesModules, req.Package)
|
||||
if err != nil {
|
||||
if err == sql.ErrNoRows {
|
||||
return nil, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return modules, nil
|
||||
}
|
||||
|
||||
func (s *Sqlite) CommentDelete(ctx context.Context, req *pb.CommentDeleteReq) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Sqlite) CommentCreate(ctx context.Context, req *pb.CommentCreateReq) (*models.Comment, error) {
|
||||
tx, err := s.db.BeginTx(ctx, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
defer func() {
|
||||
if err != nil {
|
||||
if rollbackErr := tx.Rollback(); rollbackErr != nil {
|
||||
s.logger.Error(ctx, "AddComment: unable to rollback: %v", rollbackErr)
|
||||
}
|
||||
} else {
|
||||
err = tx.Commit()
|
||||
}
|
||||
}()
|
||||
|
||||
if _, err = tx.ExecContext(ctx, queryCommentsCreate, req.Comment, req.PackageId); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
func (s *Sqlite) PackagesProcess(ctx context.Context, td time.Duration) ([]*models.Package, error) {
|
||||
var packages []*models.Package
|
||||
err := s.db.SelectContext(ctx, &packages, queryPackagesProcess, td.Seconds())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return packages, nil
|
||||
}
|
||||
|
||||
func (s *Sqlite) PackagesUpdateLastCheck(ctx context.Context, packages []*models.Package) error {
|
||||
tx, err := s.db.BeginTxx(ctx, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
for _, pkg := range packages {
|
||||
if _, err = tx.ExecContext(ctx, queryPackagesUpdateLastCheck, pkg.ID); err != nil {
|
||||
tx.Rollback()
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if err = tx.Commit(); err != nil {
|
||||
tx.Rollback()
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Sqlite) ModulesProcess(ctx context.Context, td time.Duration) ([]*models.Module, error) {
|
||||
var modules []*models.Module
|
||||
err := s.db.SelectContext(ctx, &modules, queryModulesProcess, td.Seconds())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return modules, nil
|
||||
}
|
||||
|
||||
func (s *Sqlite) PackageCreate(ctx context.Context, req *pb.PackageCreateReq) (*models.Package, error) {
|
||||
pkg := &models.Package{}
|
||||
err := s.db.GetContext(ctx, pkg, queryPackagesCreate, req.Name, req.Url)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return pkg, nil
|
||||
}
|
||||
|
||||
func (s *Sqlite) ModuleCreate(ctx context.Context, modules []*models.Module) error {
|
||||
tx, err := s.db.BeginTxx(ctx, nil)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
for _, mod := range modules {
|
||||
err = tx.GetContext(ctx, mod, queryModulesCreate, mod.Name, mod.Version)
|
||||
if err != nil {
|
||||
_ = tx.Rollback()
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
if err = tx.Commit(); err != nil {
|
||||
_ = tx.Rollback()
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *Sqlite) ModuleList(ctx context.Context, req *pb.ModuleListReq) ([]*models.Module, error) {
|
||||
var modules []*models.Module
|
||||
|
||||
err := s.db.SelectContext(ctx, &modules, queryModulesList)
|
||||
if err != nil {
|
||||
if err == sql.ErrNoRows {
|
||||
return nil, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return modules, nil
|
||||
}
|
||||
|
||||
func (s *Sqlite) CommentList(ctx context.Context, req *pb.CommentListReq) ([]*models.Comment, error) {
|
||||
var comments []*models.Comment
|
||||
|
||||
err := s.db.SelectContext(ctx, &comments, queryCommentsList, req.Package)
|
||||
if err != nil {
|
||||
if err == sql.ErrNoRows {
|
||||
return nil, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return comments, nil
|
||||
}
|
||||
|
||||
func (s *Sqlite) HandlerList(ctx context.Context, req *pb.HandlerListReq) ([]*models.Handler, error) {
|
||||
var handlers []*models.Handler
|
||||
|
||||
err := s.db.SelectContext(ctx, &handlers, queryHandlersList, req.Package)
|
||||
if err != nil {
|
||||
if err == sql.ErrNoRows {
|
||||
return nil, nil
|
||||
}
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return handlers, nil
|
||||
}
|
||||
50
internal/storage/storage.go
Normal file
50
internal/storage/storage.go
Normal file
@@ -0,0 +1,50 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"time"
|
||||
|
||||
"github.com/jmoiron/sqlx"
|
||||
"go.unistack.org/micro/v3/logger"
|
||||
"go.unistack.org/pkgdash/internal/models"
|
||||
pb "go.unistack.org/pkgdash/proto"
|
||||
)
|
||||
|
||||
func RegisterStorage(name string, fn func(logger.Logger, *sqlx.DB) interface{}) {
|
||||
storages[name] = fn
|
||||
}
|
||||
|
||||
var storages = map[string]func(logger.Logger, *sqlx.DB) interface{}{}
|
||||
|
||||
type Storage interface {
|
||||
PackageModulesCreate(ctx context.Context, pkg *models.Package, modules []*models.Module) error
|
||||
PackagesUpdateLastCheck(ctx context.Context, packages []*models.Package) error
|
||||
PackageModules(ctx context.Context, req *pb.PackageModulesReq) ([]*models.Module, error)
|
||||
ModulesProcess(ctx context.Context, td time.Duration) ([]*models.Module, error)
|
||||
PackagesProcess(ctx context.Context, td time.Duration) ([]*models.Package, error)
|
||||
PackageCreate(ctx context.Context, req *pb.PackageCreateReq) (*models.Package, error)
|
||||
HandlerList(ctx context.Context, req *pb.HandlerListReq) ([]*models.Handler, error)
|
||||
PackageList(ctx context.Context, req *pb.PackageListReq) ([]*models.Package, error)
|
||||
PackageLookup(ctx context.Context, req *pb.PackageLookupReq) (*models.Package, error)
|
||||
PackageUpdate(ctx context.Context, req *pb.PackageUpdateReq) (*models.Package, error)
|
||||
PackageDelete(ctx context.Context, req *pb.PackageDeleteReq) error
|
||||
CommentCreate(ctx context.Context, req *pb.CommentCreateReq) (*models.Comment, error)
|
||||
CommentDelete(ctx context.Context, req *pb.CommentDeleteReq) error
|
||||
CommentList(ctx context.Context, req *pb.CommentListReq) ([]*models.Comment, error)
|
||||
ModuleList(ctx context.Context, req *pb.ModuleListReq) ([]*models.Module, error)
|
||||
ModuleCreate(ctx context.Context, modules []*models.Module) error
|
||||
}
|
||||
|
||||
func NewStorage(name string, log logger.Logger, db *sqlx.DB) (Storage, error) {
|
||||
fn, ok := storages[name]
|
||||
if !ok {
|
||||
return nil, errors.New("incorrect name store")
|
||||
}
|
||||
store := fn(log, db)
|
||||
database, ok := store.(Storage)
|
||||
if !ok {
|
||||
return nil, errors.New("dont implements interface Storage")
|
||||
}
|
||||
return database, nil
|
||||
}
|
||||
69
internal/storage/storage_test.go
Normal file
69
internal/storage/storage_test.go
Normal file
@@ -0,0 +1,69 @@
|
||||
package storage
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"testing"
|
||||
|
||||
"go.unistack.org/pkgdash/internal/storage/sqlite"
|
||||
pb "go.unistack.org/pkgdash/proto"
|
||||
)
|
||||
|
||||
func TestGetModule(t *testing.T) {
|
||||
conn, err := sql.Open("sqlite3", "/Users/devstigneev_local/GolandProjects/unistack/pkgdash/identifier.sqlite")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer conn.Close()
|
||||
if err = conn.Ping(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
st := sqlite.NewStorage()
|
||||
store := st(conn, fs)
|
||||
|
||||
s, ok := store.(Storage)
|
||||
if !ok {
|
||||
t.Fatal("dont implements interface Storage")
|
||||
}
|
||||
|
||||
req := &pb.GetModuleReq{
|
||||
Id: []uint64{1, 2, 3},
|
||||
}
|
||||
|
||||
module, err := s.GetModule(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
fmt.Println(module)
|
||||
}
|
||||
|
||||
func TestGetComment(t *testing.T) {
|
||||
conn, err := sql.Open("sqlite3", "/Users/devstigneev_local/GolandProjects/unistack/pkgdash/identifier.sqlite")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer conn.Close()
|
||||
if err = conn.Ping(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
|
||||
st := sqlite.NewStorage()
|
||||
store := st(conn, fs)
|
||||
|
||||
s, ok := store.(Storage)
|
||||
if !ok {
|
||||
t.Fatal("dont implements interface Storage")
|
||||
}
|
||||
|
||||
req := &pb.GetCommentsReq{
|
||||
Id: []uint64{1, 2, 3, 15},
|
||||
}
|
||||
comments, err := s.GetComment(context.Background(), req)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
fmt.Println(comments.Decode())
|
||||
}
|
||||
226
internal/worker/worker.go
Normal file
226
internal/worker/worker.go
Normal file
@@ -0,0 +1,226 @@
|
||||
package worker
|
||||
|
||||
import (
|
||||
"context"
|
||||
"database/sql"
|
||||
"fmt"
|
||||
"io"
|
||||
"net/url"
|
||||
"os"
|
||||
"sort"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/go-git/go-git/v5"
|
||||
"github.com/go-git/go-git/v5/plumbing/filemode"
|
||||
"github.com/go-git/go-git/v5/plumbing/object"
|
||||
"github.com/go-git/go-git/v5/storage/memory"
|
||||
"github.com/pkg/errors"
|
||||
"go.unistack.org/micro/v3/logger"
|
||||
"go.unistack.org/pkgdash/internal/models"
|
||||
"go.unistack.org/pkgdash/internal/modules"
|
||||
"go.unistack.org/pkgdash/internal/storage"
|
||||
"golang.org/x/mod/modfile"
|
||||
"golang.org/x/mod/module"
|
||||
)
|
||||
|
||||
func Run(ctx context.Context, log logger.Logger, store storage.Storage, td time.Duration) {
|
||||
modTicker := time.NewTicker(5 * time.Second)
|
||||
defer modTicker.Stop()
|
||||
pkgTicker := time.NewTicker(5 * time.Second)
|
||||
defer pkgTicker.Stop()
|
||||
|
||||
var wg sync.WaitGroup
|
||||
for {
|
||||
select {
|
||||
case <-ctx.Done():
|
||||
return
|
||||
case <-pkgTicker.C:
|
||||
packages, err := store.PackagesProcess(ctx, td)
|
||||
if err != nil {
|
||||
if err != sql.ErrNoRows {
|
||||
continue
|
||||
}
|
||||
log.Fatal(ctx, "failed to get packages to process: %v", err)
|
||||
}
|
||||
wg.Add(len(packages))
|
||||
for _, pkg := range packages {
|
||||
go func(p *models.Package) {
|
||||
if err := parseModFile(ctx, log, store, p); err != nil {
|
||||
log.Error(ctx, "failed to process package %s: %v", p.Name, err)
|
||||
}
|
||||
p.LastCheck.Time = time.Now()
|
||||
wg.Done()
|
||||
}(pkg)
|
||||
}
|
||||
wg.Wait()
|
||||
if err = store.PackagesUpdateLastCheck(ctx, packages); err != nil {
|
||||
log.Error(ctx, "update packages last_check %#+v, err: %v", packages, err)
|
||||
}
|
||||
case <-modTicker.C:
|
||||
modules, err := store.ModulesProcess(ctx, td)
|
||||
if err != nil {
|
||||
if err != sql.ErrNoRows {
|
||||
continue
|
||||
}
|
||||
log.Fatal(ctx, "failed to get modules to process: %v", err)
|
||||
}
|
||||
if err := processModules(ctx, log, store, modules); err != nil {
|
||||
log.Error(ctx, "failed to process modules: %v", err)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func parseModFile(ctx context.Context, log logger.Logger, store storage.Storage, pkg *models.Package) error {
|
||||
log.Info(ctx, "process package %v", pkg)
|
||||
|
||||
u, err := url.Parse(pkg.URL)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
var rev string
|
||||
if idx := strings.Index(u.Path, "@"); idx > 0 {
|
||||
rev = u.Path[idx+1:]
|
||||
}
|
||||
|
||||
cloneOpts := &git.CloneOptions{
|
||||
URL: pkg.URL,
|
||||
Progress: os.Stdout,
|
||||
}
|
||||
|
||||
if len(rev) == 0 {
|
||||
cloneOpts.SingleBranch = true
|
||||
cloneOpts.Depth = 1
|
||||
}
|
||||
|
||||
if err = cloneOpts.Validate(); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
repo, err := git.CloneContext(ctx, memory.NewStorage(), nil, cloneOpts)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
ref, err := repo.Head()
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get head: %v", err)
|
||||
}
|
||||
|
||||
commit, err := repo.CommitObject(ref.Hash())
|
||||
if err != nil {
|
||||
return fmt.Errorf("failed to get commit: %v", err)
|
||||
}
|
||||
|
||||
tree, err := commit.Tree()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
unique := make(map[string]*models.Module)
|
||||
var mvs []module.Version
|
||||
err = tree.Files().ForEach(func(file *object.File) error {
|
||||
if file == nil {
|
||||
err = errors.New("file pointer is nil")
|
||||
log.Error(ctx, "file tree error", err)
|
||||
return err
|
||||
}
|
||||
|
||||
switch file.Mode {
|
||||
case filemode.Regular:
|
||||
if strings.HasSuffix(file.Name, "go.mod") {
|
||||
if mvs, err = parseFile(file); err != nil {
|
||||
return err
|
||||
}
|
||||
for i := range mvs {
|
||||
unique[mvs[i].Path] = &models.Module{
|
||||
Name: mvs[i].Path,
|
||||
Version: mvs[i].Version,
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
return nil
|
||||
})
|
||||
|
||||
modules := make([]*models.Module, 0, len(unique))
|
||||
for _, v := range unique {
|
||||
modules = append(modules, v)
|
||||
}
|
||||
|
||||
sort.Slice(modules, func(i, j int) bool {
|
||||
return modules[i].Name < modules[j].Name
|
||||
})
|
||||
|
||||
if err = store.PackageModulesCreate(ctx, pkg, modules); err != nil {
|
||||
log.Error(ctx, "failed to set create modules: %v", err)
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func processModules(ctx context.Context, log logger.Logger, store storage.Storage, mods []*models.Module) error {
|
||||
mvs := make(map[string]*models.Module, len(mods))
|
||||
|
||||
for _, mod := range mods {
|
||||
mvs[mod.Name] = mod
|
||||
}
|
||||
|
||||
mvsu := make([]module.Version, 0, len(mvs))
|
||||
for _, mv := range mvs {
|
||||
mvsu = append(mvsu, module.Version{Path: mv.Name, Version: mv.Version})
|
||||
}
|
||||
|
||||
modules.Updates(modules.UpdateOptions{
|
||||
Pre: false,
|
||||
Major: false,
|
||||
Cached: false,
|
||||
Modules: mvsu,
|
||||
OnUpdate: func(u modules.Update) {
|
||||
if u.Err != nil {
|
||||
log.Error(ctx, "%s: failed: %v", u.Module.Path, u.Err)
|
||||
} else {
|
||||
mvs[u.Module.Path].Version = u.Version
|
||||
}
|
||||
},
|
||||
})
|
||||
|
||||
if err := store.ModuleCreate(ctx, mods); err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
func parseFile(file *object.File) ([]module.Version, error) {
|
||||
r, err := file.Reader()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
data, err := io.ReadAll(r)
|
||||
r.Close()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
modfile, err := modfile.ParseLax("go.mod", data, nil)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
mods := make([]module.Version, 0, len(modfile.Require))
|
||||
for _, req := range modfile.Require {
|
||||
mods = append(mods, req.Mod)
|
||||
}
|
||||
|
||||
sort.Slice(mods, func(i, j int) bool {
|
||||
return mods[i].Path < mods[j].Path
|
||||
})
|
||||
|
||||
return mods, nil
|
||||
}
|
||||
Reference in New Issue
Block a user