rewrite

Signed-off-by: Vasiliy Tolstov <v.tolstov@unistack.org>

internal/service/client_git/client.go

@@ -0,0 +1,218 @@
+package client_git
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"net/url"
+	"os"
+	"sort"
+	"strings"
+
+	"github.com/go-git/go-git/v5"
+	"github.com/go-git/go-git/v5/plumbing/filemode"
+	"github.com/go-git/go-git/v5/plumbing/object"
+	"github.com/go-git/go-git/v5/storage/memory"
+	"github.com/pkg/errors"
+	"go.unistack.org/micro/v4/logger"
+	"git.unistack.org/unistack-org/pkgdash/internal"
+	"git.unistack.org/unistack-org/pkgdash/internal/models"
+	"git.unistack.org/unistack-org/pkgdash/internal/storage"
+	pb "git.unistack.org/unistack-org/pkgdash/proto"
+	"golang.org/x/mod/modfile"
+	"golang.org/x/mod/module"
+)
+
+type Client interface {
+	Run(ctx context.Context, st storage.Storage) chan *pb.PackagesCreateReq
+	IsClose() bool
+	Done() <-chan struct{}
+}
+
+type client struct {
+	worker chan *pb.PackagesCreateReq
+	closed bool
+	lock   chan struct{}
+}
+
+func NewClient(cap uint) Client {
+	return &client{
+		make(chan *pb.PackagesCreateReq, cap),
+		false,
+		make(chan struct{}),
+	}
+}
+
+func (c *client) Run(ctx context.Context, st storage.Storage) chan *pb.PackagesCreateReq {
+	go func() {
+		defer close(c.worker)
+		for {
+			select {
+			case data := <-c.worker:
+				go func() {
+					runner(ctx, st, data)
+				}()
+			case <-ctx.Done():
+				logger.Info(ctx, ctx.Err())
+				return
+			}
+		}
+	}()
+
+	return c.worker
+}
+
+func (c *client) IsClose() bool {
+	return c.closed
+}
+
+// Done for locked goroutine
+func (c *client) Done() <-chan struct{} {
+	return c.lock
+}
+
+func runner(ctx context.Context, st storage.Storage, req *pb.PackagesCreateReq) {
+	modules, err := getGoModule(ctx, req.Url)
+	if err != nil {
+		logger.Error(ctx, err)
+		return
+	}
+
+	logger.Infof(ctx, "success get list mod", modules)
+
+	if req.Modules, err = st.InsertButchModules(ctx, modules); err != nil {
+		logger.Error(ctx, err)
+		return
+	}
+
+	if err = st.PackagesCreate(ctx, req); err != nil {
+		logger.Error(ctx, err)
+	}
+}
+
+func getGoModule(ctx context.Context, gitUrl string) ([]models.Module, error) {
+	u, err := url.Parse(gitUrl)
+	if err != nil {
+		logger.Fatal(ctx, err)
+	}
+
+	var rev string
+	if idx := strings.Index(u.Path, "@"); idx > 0 {
+		rev = u.Path[idx+1:]
+	}
+
+	cloneOpts := &git.CloneOptions{
+		URL:      gitUrl,
+		Progress: os.Stdout,
+	}
+
+	if len(rev) == 0 {
+		cloneOpts.SingleBranch = true
+		cloneOpts.Depth = 1
+	}
+
+	if err = cloneOpts.Validate(); err != nil {
+		return nil, err
+	}
+
+	repo, err := git.CloneContext(ctx, memory.NewStorage(), nil, cloneOpts)
+	if err != nil {
+		return nil, err
+	}
+
+	ref, err := repo.Head()
+	if err != nil {
+		return nil, fmt.Errorf("failed to get head: %v", err)
+	}
+
+	commit, err := repo.CommitObject(ref.Hash())
+	if err != nil {
+		return nil, fmt.Errorf("failed to get commit: %v", err)
+	}
+
+	tree, err := commit.Tree()
+	if err != nil {
+		return nil, err
+	}
+
+	unique := make(map[string]models.Module)
+	var mvs []module.Version
+	err = tree.Files().ForEach(func(file *object.File) error {
+		if file == nil {
+			err = errors.New("file pointer is nil")
+			logger.Error(ctx, err)
+			return err
+		}
+
+		switch file.Mode {
+		case filemode.Regular:
+			if strings.HasSuffix(file.Name, "go.mod") {
+				if mvs, err = Direct(file); err != nil {
+					return err
+				}
+				for i := range mvs {
+					unique[mvs[i].Path] = models.Module{
+						Name:        mvs[i].Path,
+						Version:     mvs[i].Version,
+						LastVersion: mvs[i].Version,
+					}
+				}
+				internal.Updates(internal.UpdateOptions{
+					Pre:     false,
+					Major:   false,
+					Cached:  false,
+					Modules: mvs,
+					OnUpdate: func(u internal.Update) {
+						if u.Err != nil {
+							logger.Errorf(ctx, "%s: failed: %v\n", u.Module.Path, u.Err)
+						} else {
+							val := unique[u.Module.Path]
+							val.LastVersion = u.Version
+							unique[u.Module.Path] = val
+						}
+					},
+				})
+			}
+		}
+		return nil
+	})
+
+	result := make([]models.Module, 0, len(unique))
+	for _, v := range unique {
+		result = append(result, v)
+	}
+
+	sort.Slice(result, func(i, j int) bool {
+		return result[i].Name < result[j].Name
+	})
+
+	return result, err
+}
+
+func Direct(file *object.File) ([]module.Version, error) {
+	r, err := file.Reader()
+	if err != nil {
+		return nil, err
+	}
+	defer r.Close()
+	data, err := io.ReadAll(r)
+	if err != nil {
+		return nil, err
+	}
+	modfile, err := modfile.ParseLax("go.mod", data, nil)
+	if err != nil {
+		return nil, err
+	}
+	var mods []module.Version
+	for _, req := range modfile.Require {
+		// if !req.Indirect {
+		mods = append(mods, req.Mod)
+		//}
+	}
+	/*
+		sort.Slice(mods, func(i, j int) bool {
+			return mods[i].Path < mods[j].Path
+		})
+	*/
+	return mods, nil
+}

internal/service/client_git/client_test.go

@@ -0,0 +1,81 @@
+package client_git
+
+import (
+	"context"
+	"database/sql"
+	"embed"
+	"fmt"
+	"testing"
+
+	"git.unistack.org/unistack-org/pkgdash/internal/storage"
+	// "git.unistack.org/unistack-org/pkgdash/internal/storage/postgres"
+	"git.unistack.org/unistack-org/pkgdash/internal/storage/sqlite"
+	pb "git.unistack.org/unistack-org/pkgdash/proto"
+)
+
+func TestClientPG(t *testing.T) {
+	dsn := fmt.Sprintf("file:///database.db")
+	conn, err := sql.Open("sqlite", dsn)
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer conn.Close()
+	if err = conn.Ping(); err != nil {
+		t.Fatal(err)
+	}
+
+	fucntion := sqlite.NewStorage()
+	st := fucntion(conn, embed.FS{})
+	s, ok := st.(storage.Storage)
+	if !ok {
+		t.Fatal("typecast error")
+	}
+
+	ctx, cancel := context.WithCancel(context.Background())
+	_ = cancel
+	cli := NewClient(1)
+
+	ch := cli.Run(ctx, s)
+
+	data := &pb.AddPackageReq{
+		Name: "test",
+		Url:  "https://github.com/dantedenis/service_history.git",
+	}
+
+	ch <- data
+
+	<-cli.Done()
+}
+
+func TestClientLite(t *testing.T) {
+	conn, err := sql.Open("sqlite3", "../../identifier.sqlite")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer conn.Close()
+	if err = conn.Ping(); err != nil {
+		t.Fatal(err)
+	}
+
+	function := sqlite.NewStorage()
+	st := function(conn, embed.FS{})
+	s, ok := st.(storage.Storage)
+	if !ok {
+		t.Fatal("typecast error")
+	}
+
+	ctx, cancel := context.WithCancel(context.Background())
+	_ = cancel
+	cli := NewClient(1)
+
+	ch := cli.Run(ctx, s)
+
+	data := &pb.AddPackageReq{
+		Name: "test",
+		Url:  "https://github.com/dantedenis/service_history.git",
+	}
+
+	ch <- data
+
+	<-cli.Done()
+}

internal/service/service.go

@@ -0,0 +1,154 @@
+package service
+
+import (
+	"context"
+	"database/sql"
+	"net/url"
+	"strings"
+
+	httpsrv "go.unistack.org/micro-server-http/v4" // TODO
+	"go.unistack.org/micro/v4"
+	"go.unistack.org/micro/v4/config"
+	microcfg "go.unistack.org/micro/v4/config"
+	"go.unistack.org/micro/v4/logger"
+	"go.unistack.org/micro/v4/options"
+	"go.unistack.org/micro/v4/register"
+	"go.unistack.org/micro/v4/server"
+	intcfg "git.unistack.org/unistack-org/pkgdash/config"
+	"git.unistack.org/unistack-org/pkgdash/handler"
+	pb "git.unistack.org/unistack-org/pkgdash/proto"
+	"git.unistack.org/unistack-org/pkgdash/service/client_git"
+	"git.unistack.org/unistack-org/pkgdash/storage"
+)
+
+func NewService(ctx context.Context) (micro.Service, error) {
+	var reg register.Register
+
+	cfg := intcfg.NewConfig(ServiceName, Service)
+
+	cs := microcfg.NewConfig(config.Struct(cfg))
+
+	// TODO
+	mgsrv := httpsrv.NewServer(
+		options.Register(reg),
+	)
+
+	svc := micro.NewService(
+		micro.Config(cs),
+	)
+
+	h := handler.NewHandler(svc, client_git.NewClient(5))
+
+	if err := svc.Init(
+		micro.AfterStart(func(_ context.Context) error {
+			return h.Init(svc.Options().Context)
+		}),
+		micro.BeforeStart(func(ctx context.Context) error {
+			if err := config.Load(ctx, []config.Config{cs}, config.LoadOverride(true)); err != nil {
+				return err
+			}
+			if err := config.Validate(ctx, cfg); err != nil {
+				return err
+			}
+
+			if err := svc.Init(
+				micro.Name(intcfg.ServiceName),
+				micro.Version(intcfg.ServiceVersion),
+			); err != nil {
+				return err
+			}
+
+			if err := svc.Server("http").Init(
+				options.Address(cfg.Address),
+				options.Name(cfg.App.Name),
+				server.Version(cfg.App.Version),
+			); err != nil {
+				return err
+			}
+
+			return nil
+		}),
+		micro.BeforeStart(func(_ context.Context) error {
+			log := logger.NewLogger(
+				logger.WithLevel(logger.ParseLevel(cfg.LogLevel)),
+				logger.WithCallerSkipCount(3),
+			)
+			return svc.Init(micro.Logger(log))
+		}),
+		micro.BeforeStart(func(ctx context.Context) error {
+			var connstr string
+			if v, ok := cfg.StorageDSN[cfg.App.Name]; ok {
+				connstr = v
+			} else if v, ok = cfg.StorageDSN["all"]; ok {
+				connstr = v
+			}
+
+			scheme, dsn, err := storageOptions(connstr)
+			if err != nil {
+				return err
+			}
+			conn, err := connectDataBase(scheme, dsn)
+			if err != nil {
+				return err
+			}
+			store, err := storage.NewStorage(scheme, conn)
+			if err != nil {
+				return err
+			}
+			ctx = storage.InContext(ctx, store)
+			return svc.Init(micro.Context(ctx))
+		}),
+	); err != nil {
+		return nil, err
+	}
+
+	if err := pb.RegisterPkgdashServiceServer(mgsrv, h); err != nil {
+		logger.Fatalf(ctx, "failed to register handler: %v", err)
+	}
+
+	intsvc := httpsrv.NewServer(
+		server.Codec("application/json", jsoncodec.NewCodec()),
+		server.Address(cfg.Meter.Addr), server.Context(ctx),
+	)
+
+	if err := intsvc.Init(); err != nil {
+		logger.Fatalf(ctx, "failed to init http srv: %v", err)
+	}
+
+	if err := healthhandler.RegisterHealthServiceServer(intsvc, healthhandler.NewHandler()); err != nil {
+		logger.Fatalf(ctx, "failed to set http handler: %v", err)
+	}
+	if err := meterhandler.RegisterMeterServiceServer(intsvc, meterhandler.NewHandler()); err != nil {
+		logger.Fatalf(ctx, "failed to set http handler: %v", err)
+	}
+
+	if err := intsvc.Start(); err != nil {
+		logger.Fatalf(ctx, "failed to run http srv: %v", err)
+	}
+
+	return svc, nil
+}
+
+func storageOptions(dsn string) (string, string, error) {
+	u, err := url.Parse(dsn)
+	if err != nil {
+		return "", "", err
+	}
+	scheme := u.Scheme
+	if idx := strings.Index(u.Scheme, "+"); idx > 0 {
+		scheme = u.Scheme[:idx]
+		u.Scheme = u.Scheme[idx+1:]
+	}
+	return scheme, u.String(), nil
+}
+
+func connectDataBase(driverName, dsn string) (*sql.DB, error) {
+	conn, err := sql.Open(driverName, dsn)
+	if err != nil {
+		return nil, err
+	}
+	if err = conn.Ping(); err != nil {
+		return nil, err
+	}
+	return conn, err
+}