package worker import ( "context" "database/sql" "fmt" "io" "net/url" "os" "sort" "strings" "sync" "time" "git.unistack.org/unistack-org/pkgdash/internal" "git.unistack.org/unistack-org/pkgdash/internal/models" "git.unistack.org/unistack-org/pkgdash/internal/storage" "github.com/go-git/go-git/v5" "github.com/go-git/go-git/v5/plumbing/filemode" "github.com/go-git/go-git/v5/plumbing/object" "github.com/go-git/go-git/v5/storage/memory" "github.com/pkg/errors" "go.unistack.org/micro/v4/logger" "golang.org/x/mod/modfile" "golang.org/x/mod/module" ) func Run(ctx context.Context, store storage.Storage, td time.Duration) { ticker := time.NewTicker(5 * time.Second) defer ticker.Stop() var wg sync.WaitGroup for { select { case <-ctx.Done(): return case <-ticker.C: logger.Infof(ctx, "check packages to process") packages, err := store.PackagesProcess(ctx, td) logger.Infof(ctx, "check packages to process %#+v, err: %v", packages, err) if err != nil && err != sql.ErrNoRows { logger.Fatalf(ctx, "failed to get packages to process: %v", err) } wg.Add(len(packages)) for _, pkg := range packages { go func(p *models.Package) { if err := process(ctx, store, p); err != nil { logger.Errorf(ctx, "failed to process package %s: %v", p.Name, err) } wg.Done() }(pkg) } wg.Wait() } } } func process(ctx context.Context, store storage.Storage, pkg *models.Package) error { logger.Infof(ctx, "process package %v", pkg) modules, err := getGoModule(ctx, pkg.ID, pkg.URL) if err != nil { logger.Errorf(ctx, "failed to get modules: %v", err) return err } if err = store.PackagesModulesCreate(ctx, pkg, modules); err != nil { logger.Errorf(ctx, "failed to set create modules: %v", err) return err } return nil } func getGoModule(ctx context.Context, pkgID uint64, gitUrl string) ([]*models.Module, error) { u, err := url.Parse(gitUrl) if err != nil { logger.Fatal(ctx, err) } var rev string if idx := strings.Index(u.Path, "@"); idx > 0 { rev = u.Path[idx+1:] } cloneOpts := &git.CloneOptions{ URL: gitUrl, Progress: os.Stdout, } if len(rev) == 0 { cloneOpts.SingleBranch = true cloneOpts.Depth = 1 } if err = cloneOpts.Validate(); err != nil { return nil, err } repo, err := git.CloneContext(ctx, memory.NewStorage(), nil, cloneOpts) if err != nil { return nil, err } ref, err := repo.Head() if err != nil { return nil, fmt.Errorf("failed to get head: %v", err) } commit, err := repo.CommitObject(ref.Hash()) if err != nil { return nil, fmt.Errorf("failed to get commit: %v", err) } tree, err := commit.Tree() if err != nil { return nil, err } unique := make(map[string]*models.Module) var mvs []module.Version err = tree.Files().ForEach(func(file *object.File) error { if file == nil { err = errors.New("file pointer is nil") logger.Error(ctx, err) return err } switch file.Mode { case filemode.Regular: if strings.HasSuffix(file.Name, "go.mod") { if mvs, err = Direct(file); err != nil { return err } for i := range mvs { unique[mvs[i].Path] = &models.Module{ Package: pkgID, Name: mvs[i].Path, Version: mvs[i].Version, LastVersion: mvs[i].Version, } } internal.Updates(internal.UpdateOptions{ Pre: false, Major: false, Cached: false, Modules: mvs, OnUpdate: func(u internal.Update) { if u.Err != nil { logger.Errorf(ctx, "%s: failed: %v\n", u.Module.Path, u.Err) } else { val := unique[u.Module.Path] val.LastVersion = u.Version unique[u.Module.Path] = val } }, }) } } return nil }) result := make([]*models.Module, 0, len(unique)) for _, v := range unique { result = append(result, v) } sort.Slice(result, func(i, j int) bool { return result[i].Name < result[j].Name }) return result, err } func Direct(file *object.File) ([]module.Version, error) { r, err := file.Reader() if err != nil { return nil, err } defer r.Close() data, err := io.ReadAll(r) if err != nil { return nil, err } modfile, err := modfile.ParseLax("go.mod", data, nil) if err != nil { return nil, err } var mods []module.Version for _, req := range modfile.Require { // if !req.Indirect { mods = append(mods, req.Mod) //} } /* sort.Slice(mods, func(i, j int) bool { return mods[i].Path < mods[j].Path }) */ return mods, nil }