unistack-org/pkgdash
4
0
Fork 1
Code Issues 2 Pull Requests 1 Packages Projects Releases Wiki Activity

update

Browse Source 
Signed-off-by: Vasiliy Tolstov <v.tolstov@unistack.org>
This commit is contained in:
Василий Толстов
2024-12-07 02:35:30 +03:00
parent 8729d0b88e
commit c5f3fa325e
42 changed files with 1316 additions and 2009 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
internal/worker/worker.go
View File

@@ -12,20 +12,20 @@ import (
"sync"
"time"
"git.unistack.org/unistack-org/pkgdash/internal/models"
"git.unistack.org/unistack-org/pkgdash/internal/modules"
"git.unistack.org/unistack-org/pkgdash/internal/storage"
"github.com/go-git/go-git/v5"
"github.com/go-git/go-git/v5/plumbing/filemode"
"github.com/go-git/go-git/v5/plumbing/object"
"github.com/go-git/go-git/v5/storage/memory"
"github.com/pkg/errors"
"go.unistack.org/micro/v4/logger"
"go.unistack.org/micro/v3/logger"
"go.unistack.org/pkgdash/internal/models"
"go.unistack.org/pkgdash/internal/modules"
"go.unistack.org/pkgdash/internal/storage"
"golang.org/x/mod/modfile"
"golang.org/x/mod/module"
)
func Run(ctx context.Context, store storage.Storage, td time.Duration) {
func Run(ctx context.Context, log logger.Logger, store storage.Storage, td time.Duration) {
modTicker := time.NewTicker(5 * time.Second)
defer modTicker.Stop()
pkgTicker := time.NewTicker(5 * time.Second)
@@ -42,13 +42,13 @@ func Run(ctx context.Context, store storage.Storage, td time.Duration) {
if err != sql.ErrNoRows {
continue
}
logger.Fatal(ctx, "failed to get packages to process: %v", err)
log.Fatal(ctx, "failed to get packages to process: %v", err)
}
wg.Add(len(packages))
for _, pkg := range packages {
go func(p *models.Package) {
if err := parseModFile(ctx, store, p); err != nil {
logger.Error(ctx, "failed to process package %s: %v", p.Name, err)
if err := parseModFile(ctx, log, store, p); err != nil {
log.Error(ctx, "failed to process package %s: %v", p.Name, err)
}
p.LastCheck.Time = time.Now()
wg.Done()
@@ -56,7 +56,7 @@ func Run(ctx context.Context, store storage.Storage, td time.Duration) {
}
wg.Wait()
if err = store.PackagesUpdateLastCheck(ctx, packages); err != nil {
logger.Error(ctx, "update packages last_check %#+v, err: %v", packages, err)
log.Error(ctx, "update packages last_check %#+v, err: %v", packages, err)
}
case <-modTicker.C:
modules, err := store.ModulesProcess(ctx, td)
@@ -64,17 +64,17 @@ func Run(ctx context.Context, store storage.Storage, td time.Duration) {
if err != sql.ErrNoRows {
continue
}
logger.Fatal(ctx, "failed to get modules to process: %v", err)
log.Fatal(ctx, "failed to get modules to process: %v", err)
}
if err := processModules(ctx, store, modules); err != nil {
logger.Error(ctx, "failed to process modules: %v", err)
if err := processModules(ctx, log, store, modules); err != nil {
log.Error(ctx, "failed to process modules: %v", err)
}
}
}
}
func parseModFile(ctx context.Context, store storage.Storage, pkg *models.Package) error {
logger.Info(ctx, "process package %v", pkg)
func parseModFile(ctx context.Context, log logger.Logger, store storage.Storage, pkg *models.Package) error {
log.Info(ctx, "process package %v", pkg)
u, err := url.Parse(pkg.URL)
if err != nil {
@@ -125,7 +125,7 @@ func parseModFile(ctx context.Context, store storage.Storage, pkg *models.Packag
err = tree.Files().ForEach(func(file *object.File) error {
if file == nil {
err = errors.New("file pointer is nil")
logger.Error(ctx, "file tree error", err)
log.Error(ctx, "file tree error", err)
return err
}
@@ -156,14 +156,14 @@ func parseModFile(ctx context.Context, store storage.Storage, pkg *models.Packag
})
if err = store.PackageModulesCreate(ctx, pkg, modules); err != nil {
logger.Error(ctx, "failed to set create modules: %v", err)
log.Error(ctx, "failed to set create modules: %v", err)
return err
}
return nil
}
func processModules(ctx context.Context, store storage.Storage, mods []*models.Module) error {
func processModules(ctx context.Context, log logger.Logger, store storage.Storage, mods []*models.Module) error {
mvs := make(map[string]*models.Module, len(mods))
for _, mod := range mods {
@@ -182,7 +182,7 @@ func processModules(ctx context.Context, store storage.Storage, mods []*models.M
Modules: mvsu,
OnUpdate: func(u modules.Update) {
if u.Err != nil {
logger.Error(ctx, "%s: failed: %v", u.Module.Path, u.Err)
log.Error(ctx, "%s: failed: %v", u.Module.Path, u.Err)
} else {
mvs[u.Module.Path].Version = u.Version
}