# Conflicts: # cmd/pkgdashcli/main.go # go.mod # go.sum # internal/cli/init.go # internal/source/gitea/gitea.go
227 lines
4.9 KiB
Go
227 lines
4.9 KiB
Go
package worker
|
|
|
|
import (
|
|
"context"
|
|
"database/sql"
|
|
"fmt"
|
|
"io"
|
|
"net/url"
|
|
"os"
|
|
"sort"
|
|
"strings"
|
|
"sync"
|
|
"time"
|
|
|
|
"git.unistack.org/unistack-org/pkgdash/internal/models"
|
|
"git.unistack.org/unistack-org/pkgdash/internal/modules"
|
|
"git.unistack.org/unistack-org/pkgdash/internal/storage"
|
|
"github.com/go-git/go-git/v5"
|
|
"github.com/go-git/go-git/v5/plumbing/filemode"
|
|
"github.com/go-git/go-git/v5/plumbing/object"
|
|
"github.com/go-git/go-git/v5/storage/memory"
|
|
"github.com/pkg/errors"
|
|
"go.unistack.org/micro/v4/logger"
|
|
"golang.org/x/mod/modfile"
|
|
"golang.org/x/mod/module"
|
|
)
|
|
|
|
func Run(ctx context.Context, store storage.Storage, td time.Duration) {
|
|
modTicker := time.NewTicker(5 * time.Second)
|
|
defer modTicker.Stop()
|
|
pkgTicker := time.NewTicker(5 * time.Second)
|
|
defer pkgTicker.Stop()
|
|
|
|
var wg sync.WaitGroup
|
|
for {
|
|
select {
|
|
case <-ctx.Done():
|
|
return
|
|
case <-pkgTicker.C:
|
|
packages, err := store.PackagesProcess(ctx, td)
|
|
if err != nil {
|
|
if err != sql.ErrNoRows {
|
|
continue
|
|
}
|
|
logger.Fatal(ctx, "failed to get packages to process: %v", err)
|
|
}
|
|
wg.Add(len(packages))
|
|
for _, pkg := range packages {
|
|
go func(p *models.Package) {
|
|
if err := parseModFile(ctx, store, p); err != nil {
|
|
logger.Error(ctx, "failed to process package %s: %v", p.Name, err)
|
|
}
|
|
p.LastCheck.Time = time.Now()
|
|
wg.Done()
|
|
}(pkg)
|
|
}
|
|
wg.Wait()
|
|
if err = store.PackagesUpdateLastCheck(ctx, packages); err != nil {
|
|
logger.Error(ctx, "update packages last_check %#+v, err: %v", packages, err)
|
|
}
|
|
case <-modTicker.C:
|
|
modules, err := store.ModulesProcess(ctx, td)
|
|
if err != nil {
|
|
if err != sql.ErrNoRows {
|
|
continue
|
|
}
|
|
logger.Fatal(ctx, "failed to get modules to process: %v", err)
|
|
}
|
|
if err := processModules(ctx, store, modules); err != nil {
|
|
logger.Error(ctx, "failed to process modules: %v", err)
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
func parseModFile(ctx context.Context, store storage.Storage, pkg *models.Package) error {
|
|
logger.Info(ctx, "process package %v", pkg)
|
|
|
|
u, err := url.Parse(pkg.URL)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
var rev string
|
|
if idx := strings.Index(u.Path, "@"); idx > 0 {
|
|
rev = u.Path[idx+1:]
|
|
}
|
|
|
|
cloneOpts := &git.CloneOptions{
|
|
URL: pkg.URL,
|
|
Progress: os.Stdout,
|
|
}
|
|
|
|
if len(rev) == 0 {
|
|
cloneOpts.SingleBranch = true
|
|
cloneOpts.Depth = 1
|
|
}
|
|
|
|
if err = cloneOpts.Validate(); err != nil {
|
|
return err
|
|
}
|
|
|
|
repo, err := git.CloneContext(ctx, memory.NewStorage(), nil, cloneOpts)
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
ref, err := repo.Head()
|
|
if err != nil {
|
|
return fmt.Error("failed to get head: %v", err)
|
|
}
|
|
|
|
commit, err := repo.CommitObject(ref.Hash())
|
|
if err != nil {
|
|
return fmt.Error("failed to get commit: %v", err)
|
|
}
|
|
|
|
tree, err := commit.Tree()
|
|
if err != nil {
|
|
return err
|
|
}
|
|
|
|
unique := make(map[string]*models.Module)
|
|
var mvs []module.Version
|
|
err = tree.Files().ForEach(func(file *object.File) error {
|
|
if file == nil {
|
|
err = errors.New("file pointer is nil")
|
|
logger.Error(ctx, "file tree error", err)
|
|
return err
|
|
}
|
|
|
|
switch file.Mode {
|
|
case filemode.Regular:
|
|
if strings.HasSuffix(file.Name, "go.mod") {
|
|
if mvs, err = parseFile(file); err != nil {
|
|
return err
|
|
}
|
|
for i := range mvs {
|
|
unique[mvs[i].Path] = &models.Module{
|
|
Name: mvs[i].Path,
|
|
Version: mvs[i].Version,
|
|
}
|
|
}
|
|
}
|
|
}
|
|
return nil
|
|
})
|
|
|
|
modules := make([]*models.Module, 0, len(unique))
|
|
for _, v := range unique {
|
|
modules = append(modules, v)
|
|
}
|
|
|
|
sort.Slice(modules, func(i, j int) bool {
|
|
return modules[i].Name < modules[j].Name
|
|
})
|
|
|
|
if err = store.PackageModulesCreate(ctx, pkg, modules); err != nil {
|
|
logger.Error(ctx, "failed to set create modules: %v", err)
|
|
return err
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func processModules(ctx context.Context, store storage.Storage, mods []*models.Module) error {
|
|
mvs := make(map[string]*models.Module, len(mods))
|
|
|
|
for _, mod := range mods {
|
|
mvs[mod.Name] = mod
|
|
}
|
|
|
|
mvsu := make([]module.Version, 0, len(mvs))
|
|
for _, mv := range mvs {
|
|
mvsu = append(mvsu, module.Version{Path: mv.Name, Version: mv.Version})
|
|
}
|
|
|
|
modules.Updates(modules.UpdateOptions{
|
|
Pre: false,
|
|
Major: false,
|
|
Cached: false,
|
|
Modules: mvsu,
|
|
OnUpdate: func(u modules.Update) {
|
|
if u.Err != nil {
|
|
logger.Error(ctx, "%s: failed: %v", u.Module.Path, u.Err)
|
|
} else {
|
|
mvs[u.Module.Path].Version = u.Version
|
|
}
|
|
},
|
|
})
|
|
|
|
if err := store.ModuleCreate(ctx, mods); err != nil {
|
|
return err
|
|
}
|
|
|
|
return nil
|
|
}
|
|
|
|
func parseFile(file *object.File) ([]module.Version, error) {
|
|
r, err := file.Reader()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
data, err := io.ReadAll(r)
|
|
r.Close()
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
modfile, err := modfile.ParseLax("go.mod", data, nil)
|
|
if err != nil {
|
|
return nil, err
|
|
}
|
|
|
|
mods := make([]module.Version, 0, len(modfile.Require))
|
|
for _, req := range modfile.Require {
|
|
mods = append(mods, req.Mod)
|
|
}
|
|
|
|
sort.Slice(mods, func(i, j int) bool {
|
|
return mods[i].Path < mods[j].Path
|
|
})
|
|
|
|
return mods, nil
|
|
}
|