package client_git import ( "context" "fmt" "github.com/pkg/errors" "go.unistack.org/unistack-org/pkgdash/internal" "go.unistack.org/unistack-org/pkgdash/models" "io" "net/url" "os" "sort" "strings" "github.com/go-git/go-git/v5" "github.com/go-git/go-git/v5/plumbing/filemode" "github.com/go-git/go-git/v5/plumbing/object" "github.com/go-git/go-git/v5/storage/memory" "go.unistack.org/micro/v3/logger" pb "go.unistack.org/unistack-org/pkgdash/proto/go_generate" "go.unistack.org/unistack-org/pkgdash/storage" "golang.org/x/mod/modfile" "golang.org/x/mod/module" ) type Client interface { Run(ctx context.Context, st storage.Storage) chan *pb.AddPackageRsp IsClose() bool Done() <-chan struct{} } type client struct { worker chan *pb.AddPackageRsp closed bool lock chan struct{} } func NewClient(cap uint) Client { return &client{ make(chan *pb.AddPackageRsp, cap), false, make(chan struct{}), } } func (c *client) Run(ctx context.Context, st storage.Storage) chan *pb.AddPackageRsp { go func() { defer close(c.worker) for { select { case data := <-c.worker: go func() { runner(ctx, st, data) }() case <-ctx.Done(): logger.Info(ctx, ctx.Err()) return } } }() return c.worker } func (c *client) IsClose() bool { return c.closed } // Done for locked goroutine func (c *client) Done() <-chan struct{} { return c.lock } func runner(ctx context.Context, st storage.Storage, rsp *pb.AddPackageRsp) { modules, err := getGoModule(ctx, rsp.Url.Value) if err != nil { logger.Error(ctx, err) return } logger.Infof(ctx, "success get list mod", modules) if rsp.Modules, err = st.InsertButchModules(ctx, modules); err != nil { logger.Error(ctx, err) return } if err = st.AddPackage(ctx, rsp); err != nil { logger.Error(ctx, err) } } func getGoModule(ctx context.Context, gitUrl string) ([]models.Module, error) { u, err := url.Parse(gitUrl) if err != nil { logger.Fatal(ctx, err) } var rev string if idx := strings.Index(u.Path, "@"); idx > 0 { rev = u.Path[idx+1:] } cloneOpts := &git.CloneOptions{ URL: gitUrl, Progress: os.Stdout, } if len(rev) == 0 { cloneOpts.SingleBranch = true cloneOpts.Depth = 1 } if err = cloneOpts.Validate(); err != nil { return nil, err } repo, err := git.CloneContext(ctx, memory.NewStorage(), nil, cloneOpts) if err != nil { return nil, err } ref, err := repo.Head() if err != nil { return nil, fmt.Errorf("failed to get head: %v", err) } commit, err := repo.CommitObject(ref.Hash()) if err != nil { return nil, fmt.Errorf("failed to get commit: %v", err) } tree, err := commit.Tree() if err != nil { return nil, err } unique := make(map[string]models.Module) var mvs []module.Version err = tree.Files().ForEach(func(file *object.File) error { if file == nil { err = errors.New("file pointer is nil") logger.Error(ctx, err) return err } switch file.Mode { case filemode.Regular: if strings.HasSuffix(file.Name, "go.mod") { if mvs, err = Direct(file); err != nil { return err } for i := range mvs { unique[mvs[i].Path] = models.Module{ Name: mvs[i].Path, Version: mvs[i].Version, LastVersion: mvs[i].Version, } } internal.Updates(internal.UpdateOptions{ Pre: false, Major: false, Cached: false, Modules: mvs, OnUpdate: func(u internal.Update) { if u.Err != nil { logger.Errorf(ctx, "%s: failed: %v\n", u.Module.Path, u.Err) } else { val := unique[u.Module.Path] val.LastVersion = u.Version unique[u.Module.Path] = val } }, }) } } return nil }) result := make([]models.Module, 0, len(unique)) for _, v := range unique { result = append(result, v) } sort.Slice(result, func(i, j int) bool { return result[i].Name < result[j].Name }) return result, err } func Direct(file *object.File) ([]module.Version, error) { r, err := file.Reader() if err != nil { return nil, err } defer r.Close() data, err := io.ReadAll(r) if err != nil { return nil, err } modfile, err := modfile.ParseLax("go.mod", data, nil) if err != nil { return nil, err } var mods []module.Version for _, req := range modfile.Require { // if !req.Indirect { mods = append(mods, req.Mod) //} } /* sort.Slice(mods, func(i, j int) bool { return mods[i].Path < mods[j].Path }) */ return mods, nil }