updates

Signed-off-by: Vasiliy Tolstov <v.tolstov@unistack.org>

internal/worker/worker.go

@@ -12,8 +12,8 @@ import (
 	"sync"
 	"time"
 
-	"git.unistack.org/unistack-org/pkgdash/internal"
 	"git.unistack.org/unistack-org/pkgdash/internal/models"
+	"git.unistack.org/unistack-org/pkgdash/internal/modules"
 	"git.unistack.org/unistack-org/pkgdash/internal/storage"
 	"github.com/go-git/go-git/v5"
 	"github.com/go-git/go-git/v5/plumbing/filemode"
@@ -26,63 +26,68 @@ import (
 )
 
 func Run(ctx context.Context, store storage.Storage, td time.Duration) {
-	ticker := time.NewTicker(5 * time.Second)
-	defer ticker.Stop()
+	modTicker := time.NewTicker(5 * time.Second)
+	defer modTicker.Stop()
+	pkgTicker := time.NewTicker(5 * time.Second)
+	defer pkgTicker.Stop()
+
 	var wg sync.WaitGroup
 	for {
 		select {
 		case <-ctx.Done():
 			return
-		case <-ticker.C:
-			logger.Infof(ctx, "check packages to process")
+		case <-pkgTicker.C:
 			packages, err := store.PackagesProcess(ctx, td)
-			logger.Infof(ctx, "check packages to process %#+v, err: %v", packages, err)
-			if err != nil && err != sql.ErrNoRows {
+			if err != nil {
+				if err != sql.ErrNoRows {
+					continue
+				}
 				logger.Fatalf(ctx, "failed to get packages to process: %v", err)
 			}
 			wg.Add(len(packages))
 			for _, pkg := range packages {
 				go func(p *models.Package) {
-					if err := process(ctx, store, p); err != nil {
+					if err := parseModFile(ctx, store, p); err != nil {
 						logger.Errorf(ctx, "failed to process package %s: %v", p.Name, err)
 					}
+					p.LastCheck.Time = time.Now()
 					wg.Done()
 				}(pkg)
 			}
 			wg.Wait()
+			if err = store.PackagesUpdateLastCheck(ctx, packages); err != nil {
+				logger.Errorf(ctx, "update packages last_check %#+v, err: %v", packages, err)
+			}
+		case <-modTicker.C:
+			modules, err := store.ModulesProcess(ctx, td)
+			if err != nil {
+				if err != sql.ErrNoRows {
+					continue
+				}
+				logger.Fatalf(ctx, "failed to get modules to process: %v", err)
+			}
+			if err := processModules(ctx, store, modules); err != nil {
+				logger.Errorf(ctx, "failed to process modules: %v", err)
+			}
 		}
 	}
 }
 
-func process(ctx context.Context, store storage.Storage, pkg *models.Package) error {
+func parseModFile(ctx context.Context, store storage.Storage, pkg *models.Package) error {
 	logger.Infof(ctx, "process package %v", pkg)
-	modules, err := getGoModule(ctx, pkg.ID, pkg.URL)
+
+	u, err := url.Parse(pkg.URL)
 	if err != nil {
-		logger.Errorf(ctx, "failed to get modules: %v", err)
 		return err
 	}
 
-	if err = store.PackagesModulesCreate(ctx, pkg, modules); err != nil {
-		logger.Errorf(ctx, "failed to set create modules: %v", err)
-		return err
-	}
-
-	return nil
-}
-
-func getGoModule(ctx context.Context, pkgID uint64, gitUrl string) ([]*models.Module, error) {
-	u, err := url.Parse(gitUrl)
-	if err != nil {
-		logger.Fatal(ctx, err)
-	}
-
 	var rev string
 	if idx := strings.Index(u.Path, "@"); idx > 0 {
 		rev = u.Path[idx+1:]
 	}
 
 	cloneOpts := &git.CloneOptions{
-		URL:      gitUrl,
+		URL:      pkg.URL,
 		Progress: os.Stdout,
 	}
 
@@ -92,27 +97,27 @@ func getGoModule(ctx context.Context, pkgID uint64, gitUrl string) ([]*models.Mo
 	}
 
 	if err = cloneOpts.Validate(); err != nil {
-		return nil, err
+		return err
 	}
 
 	repo, err := git.CloneContext(ctx, memory.NewStorage(), nil, cloneOpts)
 	if err != nil {
-		return nil, err
+		return err
 	}
 
 	ref, err := repo.Head()
 	if err != nil {
-		return nil, fmt.Errorf("failed to get head: %v", err)
+		return fmt.Errorf("failed to get head: %v", err)
 	}
 
 	commit, err := repo.CommitObject(ref.Hash())
 	if err != nil {
-		return nil, fmt.Errorf("failed to get commit: %v", err)
+		return fmt.Errorf("failed to get commit: %v", err)
 	}
 
 	tree, err := commit.Tree()
 	if err != nil {
-		return nil, err
+		return err
 	}
 
 	unique := make(map[string]*models.Module)
@@ -127,73 +132,95 @@ func getGoModule(ctx context.Context, pkgID uint64, gitUrl string) ([]*models.Mo
 		switch file.Mode {
 		case filemode.Regular:
 			if strings.HasSuffix(file.Name, "go.mod") {
-				if mvs, err = Direct(file); err != nil {
+				if mvs, err = parseFile(file); err != nil {
 					return err
 				}
 				for i := range mvs {
 					unique[mvs[i].Path] = &models.Module{
-						Package:     pkgID,
-						Name:        mvs[i].Path,
-						Version:     mvs[i].Version,
-						LastVersion: mvs[i].Version,
+						Name:    mvs[i].Path,
+						Version: mvs[i].Version,
 					}
 				}
-				internal.Updates(internal.UpdateOptions{
-					Pre:     false,
-					Major:   false,
-					Cached:  false,
-					Modules: mvs,
-					OnUpdate: func(u internal.Update) {
-						if u.Err != nil {
-							logger.Errorf(ctx, "%s: failed: %v\n", u.Module.Path, u.Err)
-						} else {
-							val := unique[u.Module.Path]
-							val.LastVersion = u.Version
-							unique[u.Module.Path] = val
-						}
-					},
-				})
 			}
 		}
 		return nil
 	})
 
-	result := make([]*models.Module, 0, len(unique))
+	modules := make([]*models.Module, 0, len(unique))
 	for _, v := range unique {
-		result = append(result, v)
+		modules = append(modules, v)
 	}
 
-	sort.Slice(result, func(i, j int) bool {
-		return result[i].Name < result[j].Name
+	sort.Slice(modules, func(i, j int) bool {
+		return modules[i].Name < modules[j].Name
 	})
 
-	return result, err
+	if err = store.PackagesModulesCreate(ctx, pkg, modules); err != nil {
+		logger.Errorf(ctx, "failed to set create modules: %v", err)
+		return err
+	}
+
+	return nil
 }
 
-func Direct(file *object.File) ([]module.Version, error) {
+func processModules(ctx context.Context, store storage.Storage, mods []*models.Module) error {
+	mvs := make(map[string]*models.Module, len(mods))
+
+	for _, mod := range mods {
+		mvs[mod.Name] = mod
+	}
+
+	mvsu := make([]module.Version, 0, len(mvs))
+	for _, mv := range mvs {
+		mvsu = append(mvsu, module.Version{Path: mv.Name, Version: mv.Version})
+	}
+
+	modules.Updates(modules.UpdateOptions{
+		Pre:     false,
+		Major:   false,
+		Cached:  false,
+		Modules: mvsu,
+		OnUpdate: func(u modules.Update) {
+			if u.Err != nil {
+				logger.Errorf(ctx, "%s: failed: %v", u.Module.Path, u.Err)
+			} else {
+				mvs[u.Module.Path].Version = u.Version
+			}
+		},
+	})
+
+	if err := store.ModulesCreate(ctx, mods); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func parseFile(file *object.File) ([]module.Version, error) {
 	r, err := file.Reader()
 	if err != nil {
 		return nil, err
 	}
-	defer r.Close()
+
 	data, err := io.ReadAll(r)
+	r.Close()
 	if err != nil {
 		return nil, err
 	}
+
 	modfile, err := modfile.ParseLax("go.mod", data, nil)
 	if err != nil {
 		return nil, err
 	}
-	var mods []module.Version
+
+	mods := make([]module.Version, 0, len(modfile.Require))
 	for _, req := range modfile.Require {
-		// if !req.Indirect {
 		mods = append(mods, req.Mod)
-		//}
 	}
-	/*
-		sort.Slice(mods, func(i, j int) bool {
-			return mods[i].Path < mods[j].Path
-		})
-	*/
+
+	sort.Slice(mods, func(i, j int) bool {
+		return mods[i].Path < mods[j].Path
+	})
+
 	return mods, nil
 }