pkgdash/service/client_git/client.go

package client_git

import (
	"context"
	"fmt"
	"io"
	"net/url"
	"os"
	"sort"
	"strings"

	"github.com/go-git/go-git/v5"
	"github.com/go-git/go-git/v5/plumbing/filemode"
	"github.com/go-git/go-git/v5/plumbing/object"
	"github.com/go-git/go-git/v5/storage/memory"
	"github.com/pkg/errors"
	"go.unistack.org/micro/v4/logger"
	"go.unistack.org/unistack-org/pkgdash/internal"
	"go.unistack.org/unistack-org/pkgdash/models"
	pb "go.unistack.org/unistack-org/pkgdash/proto"
	"go.unistack.org/unistack-org/pkgdash/storage"
	"golang.org/x/mod/modfile"
	"golang.org/x/mod/module"
)

type Client interface {
	Run(ctx context.Context, st storage.Storage) chan *pb.AddPackageReq
	IsClose() bool
	Done() <-chan struct{}
}

type client struct {
	worker chan *pb.AddPackageReq
	closed bool
	lock   chan struct{}
}

func NewClient(cap uint) Client {
	return &client{
		make(chan *pb.AddPackageReq, cap),
		false,
		make(chan struct{}),
	}
}

func (c *client) Run(ctx context.Context, st storage.Storage) chan *pb.AddPackageReq {
	go func() {
		defer close(c.worker)
		for {
			select {
			case data := <-c.worker:
				go func() {
					runner(ctx, st, data)
				}()
			case <-ctx.Done():
				logger.Info(ctx, ctx.Err())
				return
			}
		}
	}()

	return c.worker
}

func (c *client) IsClose() bool {
	return c.closed
}

// Done for locked goroutine
func (c *client) Done() <-chan struct{} {
	return c.lock
}

func runner(ctx context.Context, st storage.Storage, req *pb.AddPackageReq) {
	modules, err := getGoModule(ctx, req.Url)
	if err != nil {
		logger.Error(ctx, err)
		return
	}

	logger.Infof(ctx, "success get list mod", modules)

	if req.Modules, err = st.InsertButchModules(ctx, modules); err != nil {
		logger.Error(ctx, err)
		return
	}

	if err = st.AddPackage(ctx, req); err != nil {
		logger.Error(ctx, err)
	}
}

func getGoModule(ctx context.Context, gitUrl string) ([]models.Module, error) {
	u, err := url.Parse(gitUrl)
	if err != nil {
		logger.Fatal(ctx, err)
	}

	var rev string
	if idx := strings.Index(u.Path, "@"); idx > 0 {
		rev = u.Path[idx+1:]
	}

	cloneOpts := &git.CloneOptions{
		URL:      gitUrl,
		Progress: os.Stdout,
	}

	if len(rev) == 0 {
		cloneOpts.SingleBranch = true
		cloneOpts.Depth = 1
	}

	if err = cloneOpts.Validate(); err != nil {
		return nil, err
	}

	repo, err := git.CloneContext(ctx, memory.NewStorage(), nil, cloneOpts)
	if err != nil {
		return nil, err
	}

	ref, err := repo.Head()
	if err != nil {
		return nil, fmt.Errorf("failed to get head: %v", err)
	}

	commit, err := repo.CommitObject(ref.Hash())
	if err != nil {
		return nil, fmt.Errorf("failed to get commit: %v", err)
	}

	tree, err := commit.Tree()
	if err != nil {
		return nil, err
	}

	unique := make(map[string]models.Module)
	var mvs []module.Version
	err = tree.Files().ForEach(func(file *object.File) error {
		if file == nil {
			err = errors.New("file pointer is nil")
			logger.Error(ctx, err)
			return err
		}

		switch file.Mode {
		case filemode.Regular:
			if strings.HasSuffix(file.Name, "go.mod") {
				if mvs, err = Direct(file); err != nil {
					return err
				}
				for i := range mvs {
					unique[mvs[i].Path] = models.Module{
						Name:        mvs[i].Path,
						Version:     mvs[i].Version,
						LastVersion: mvs[i].Version,
					}
				}
				internal.Updates(internal.UpdateOptions{
					Pre:     false,
					Major:   false,
					Cached:  false,
					Modules: mvs,
					OnUpdate: func(u internal.Update) {
						if u.Err != nil {
							logger.Errorf(ctx, "%s: failed: %v\n", u.Module.Path, u.Err)
						} else {
							val := unique[u.Module.Path]
							val.LastVersion = u.Version
							unique[u.Module.Path] = val
						}
					},
				})
			}
		}
		return nil
	})

	result := make([]models.Module, 0, len(unique))
	for _, v := range unique {
		result = append(result, v)
	}

	sort.Slice(result, func(i, j int) bool {
		return result[i].Name < result[j].Name
	})

	return result, err
}

func Direct(file *object.File) ([]module.Version, error) {
	r, err := file.Reader()
	if err != nil {
		return nil, err
	}
	defer r.Close()
	data, err := io.ReadAll(r)
	if err != nil {
		return nil, err
	}
	modfile, err := modfile.ParseLax("go.mod", data, nil)
	if err != nil {
		return nil, err
	}
	var mods []module.Version
	for _, req := range modfile.Require {
		// if !req.Indirect {
		mods = append(mods, req.Mod)
		//}
	}
	/*
		sort.Slice(mods, func(i, j int) bool {
			return mods[i].Path < mods[j].Path
		})
	*/
	return mods, nil
}