#19 reset HEAD

2024-12-18 21:16:11 +03:00
parent 3d8504bf80
commit 8e633fe83f
156 changed files with 159196 additions and 42 deletions
--- a/internal/storage/postgres_tmp/queries.go
+++ b/internal/storage/postgres_tmp/queries.go
@@ -0,0 +1,34 @@
+//go:build ignore
+
+package postgres
+
+const (
+	queryListPackage = `
+select 
+    id,
+    name,
+    url,
+	comments
+	--modules,
+	--issues,
+    from package;
+`
+	queryAddComment = `
+with insert_comm as (
+    insert into comment(text) values ($1) returning id
+)
+update package set comments = array_append(comments, (select * from insert_comm)) where id=$2;
+`
+	queryAddPackage = `
+insert into package(name, url, modules) values ($1, $2, $3);
+`
+	queryInsMsgGetIDs = `
+insert into module(name, version, last_version)  values
+%s
+returning id;
+`
+	queryGetModule = `
+select id, name, version, last_version from module
+where id in %s ;
+`
+)
--- a/internal/storage/postgres_tmp/storage.go
+++ b/internal/storage/postgres_tmp/storage.go
@@ -0,0 +1,235 @@
+//go:build ignore
+
+package postgres
+
+import (
+	"context"
+	"database/sql"
+	"embed"
+	"errors"
+	"fmt"
+	"strings"
+
+	"github.com/golang-migrate/migrate/v3"
+	mpgx "github.com/golang-migrate/migrate/v4/database/pgx"
+	"github.com/golang-migrate/migrate/v4/source/iofs"
+	"github.com/lib/pq"
+	"go.unistack.org/micro/v3/logger"
+	"go.unistack.org/pkgdash/internal/config"
+	"go.unistack.org/pkgdash/internal/models"
+	pb "go.unistack.org/pkgdash/proto"
+)
+
+const (
+	pathMigration = `migrations/postgres`
+)
+
+type Postgres struct {
+	db *sql.DB
+	fs embed.FS
+}
+
+func NewStorage() func(*sql.DB, embed.FS) interface{} {
+	return func(db *sql.DB, fs embed.FS) interface{} {
+		return &Postgres{db: db, fs: fs}
+	}
+}
+
+func (s *Postgres) MigrateUp() error {
+	driver, err := mpgx.WithInstance(s.db, &mpgx.Config{
+		MigrationsTable: mpgx.DefaultMigrationsTable,
+		DatabaseName:    config.ServiceName,
+	})
+	if err != nil {
+		return err
+	}
+	source, err := iofs.New(s.fs, pathMigration)
+	if err != nil {
+		return err
+	}
+
+	// TODO: pass own logger
+	m, err := migrate.NewWithInstance("fs", source, config.ServiceName, driver)
+	if err != nil {
+		return err
+	}
+
+	if err = m.Up(); err != nil && !errors.Is(err, migrate.ErrNoChange) {
+		return err
+	}
+
+	return nil
+}
+
+func (s *Postgres) MigrateDown() error {
+	driver, err := mpgx.WithInstance(s.db, &mpgx.Config{
+		MigrationsTable: mpgx.DefaultMigrationsTable,
+		DatabaseName:    config.ServiceName,
+	})
+	if err != nil {
+		return err
+	}
+	source, err := iofs.New(s.fs, pathMigration)
+	if err != nil {
+		return err
+	}
+
+	// TODO: pass own logger
+	m, err := migrate.NewWithInstance("fs", source, config.ServiceName, driver)
+	if err != nil {
+		return err
+	}
+
+	if err = m.Down(); err != nil && !errors.Is(err, migrate.ErrNoChange) {
+		return err
+	}
+
+	return nil
+}
+
+func (s *Postgres) PackagesUpdate(ctx context.Context, req *pb.PackagesUpdateReq) error {
+	panic("need implement")
+}
+
+func (s *Postgres) PackagesList(ctx context.Context, req *pb.PackagesListReq) (models.ListPackage, error) {
+	rows, err := s.db.QueryContext(ctx, queryListPackage)
+	if err != nil {
+		return nil, err
+	}
+
+	defer func() {
+		if err = rows.Close(); err != nil {
+			return
+		}
+		err = rows.Err()
+	}()
+
+	result := make([]*models.Package, 0)
+	for rows.Next() {
+		tmp := &models.Package{}
+		if err = rows.Scan(
+			&tmp.ID,
+			&tmp.Name,
+			&tmp.URL,
+			pq.Array(&tmp.Comments),
+		); err != nil {
+			return nil, err
+		}
+	}
+
+	return result, err
+}
+
+func (s *Postgres) CommentsCreate(ctx context.Context, req *pb.CommentsCreateReq) error {
+	tx, err := s.db.BeginTx(ctx, nil)
+	if err != nil {
+		return err
+	}
+
+	defer func() {
+		if err != nil {
+			if rollbackErr := tx.Rollback(); rollbackErr != nil {
+				logger.Error(ctx, "AddComment: unable to rollback: %v", rollbackErr)
+			}
+		} else {
+			err = tx.Commit()
+		}
+	}()
+
+	res, err := tx.ExecContext(ctx, queryAddComment, req.Text, req.PackageId)
+	if err != nil {
+		return err
+	}
+
+	if aff, affErr := res.RowsAffected(); err != nil {
+		err = affErr
+	} else if aff == 0 {
+		err = errors.New("rows affected is 0")
+	}
+
+	return err
+}
+
+func (s *Postgres) PackagesCreate(ctx context.Context, req *pb.PackagesCreateReq) error {
+	tx, err := s.db.BeginTx(ctx, nil)
+	if err != nil {
+		return err
+	}
+
+	defer func() {
+		if err != nil {
+			if rollbackErr := tx.Rollback(); rollbackErr != nil {
+				logger.Error(ctx, "AddPackage: unable to rollback: %v", rollbackErr)
+			}
+		} else {
+			err = tx.Commit()
+		}
+	}()
+
+	res, err := tx.ExecContext(ctx, queryAddPackage, req.Name, req.Url, pq.Array(req.Modules))
+	if err != nil {
+		return err
+	}
+
+	if aff, affErr := res.RowsAffected(); err != nil {
+		err = affErr
+	} else if aff == 0 {
+		err = errors.New("rows affected is 0")
+	}
+
+	return err
+}
+
+func (s *Postgres) InsertButchModules(ctx context.Context, req []models.Module) ([]uint64, error) {
+	tx, err := s.db.BeginTx(ctx, nil)
+	if err != nil {
+		return nil, err
+	}
+	defer func() {
+		if err != nil {
+			if rollbackErr := tx.Rollback(); rollbackErr != nil {
+				logger.Error(ctx, "AddPackage: unable to rollback: %v", rollbackErr)
+			}
+		} else {
+			err = tx.Commit()
+		}
+	}()
+
+	query := generateQuery(req)
+
+	rows, err := tx.QueryContext(ctx, query)
+	if err != nil {
+		return nil, err
+	}
+	defer func() {
+		if err = rows.Close(); err != nil {
+			return
+		}
+		err = rows.Err()
+	}()
+
+	result := make([]uint64, 0)
+	for rows.Next() {
+		tmp := uint64(0)
+		if err = rows.Scan(&tmp); err != nil {
+			return nil, err
+		}
+
+		result = append(result, tmp)
+	}
+
+	return result, err
+}
+
+func generateQuery(rsp []models.Module) string {
+	const pattern = `%c('%s', '%s', '%s')`
+	build := strings.Builder{}
+	comma := ' '
+	for i := range rsp {
+		str := fmt.Sprintf(pattern, comma, rsp[i].Name, rsp[i].Version, rsp[i].LastVersion)
+		build.WriteString(str)
+		comma = ','
+	}
+
+	return fmt.Sprintf(queryInsMsgGetIDs, build.String())
+}
--- a/internal/storage/postgres_tmp/storage_test.go
+++ b/internal/storage/postgres_tmp/storage_test.go
@@ -0,0 +1,40 @@
+//go:build ignore
+
+package postgres
+
+import (
+	"fmt"
+	"testing"
+
+	"go.unistack.org/pkgdash/internal/models"
+)
+
+func TestGenerate(t *testing.T) {
+	m := []models.Module{
+		{
+			ID:          1,
+			Name:        "test",
+			Version:     "1.2.3",
+			Package:     2,
+			LastVersion: "23.31",
+		},
+		{
+			ID:          1,
+			Name:        "321test",
+			Version:     "1.3",
+			Package:     4,
+			LastVersion: "2111.31",
+		},
+		{
+			ID:          1,
+			Name:        "testabcd",
+			Version:     "1.2.3",
+			Package:     2,
+			LastVersion: "153453.31",
+		},
+	}
+
+	str := generateQuery(m)
+
+	fmt.Println(str)
+}
--- a/internal/storage/sqlite/queries.go
+++ b/internal/storage/sqlite/queries.go
@@ -0,0 +1,19 @@
+package sqlite
+
+const (
+	queryPackageModulesCount     = `update packages set modules = $2 where id = $1;`
+	queryPackagesModulesCreate   = `insert into packages_modules as pm (package, module) values ($1, $2) on conflict (package,module) do nothing;`
+	queryPackagesUpdateLastCheck = `update packages set last_check = CURRENT_TIMESTAMP where id = $1;`
+	queryPackagesModules         = `select modules.id, modules.name, modules.version from modules left join packages_modules on modules.id = packages_modules.module left join packages on packages.id = packages_modules.package where packages_modules.package = $1;`
+	queryPackagesProcess         = `select id, name, url, comments, modules, issues, created, updated, last_check from packages where ROUND((JULIANDAY(CURRENT_TIMESTAMP) - JULIANDAY(last_check)) * 86400) > $1 or last_check is NULL`
+	queryModulesProcess          = `select id, name, version, last_check from modules where ROUND((JULIANDAY(CURRENT_TIMESTAMP) - JULIANDAY(last_check)) * 86400) > $1 or last_check is NULL`
+	queryPackagesModulesCount    = `update packages set modules = $2, last_check = CURRENT_TIMESTAMP where id = $1;`
+	queryPackagesList            = `select id, name, url, comments, modules, issues, created, updated from packages;`
+	queryPackagesLookup          = `select id, name, url, comments, modules, issues, created, updated from packages where id = $1;`
+	queryCommentsCreate          = `insert into comments (comment) values ($1) returning id;`
+	queryPackagesCreate          = `insert into packages as p (name, url) values ($1, $2) on conflict (url) do update set name = p.name returning *;`
+	queryModulesList             = `select id, name, version from modules;`
+	queryModulesCreate           = `insert into modules as m (name, version) values ($1, $2) on conflict (name,version) do update set last_check = CURRENT_TIMESTAMP returning *;`
+	queryCommentsList            = `select id, text, created, updated from comments where package = $1;`
+	queryHandlersList            = `select id, name, coverage from handlers where package = $1;`
+)
--- a/internal/storage/sqlite/storage.go
+++ b/internal/storage/sqlite/storage.go
@@ -0,0 +1,254 @@
+package sqlite
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+	"time"
+
+	"github.com/jmoiron/sqlx"
+	"go.unistack.org/micro/v3/logger"
+	"go.unistack.org/pkgdash/internal/models"
+	"go.unistack.org/pkgdash/internal/storage"
+	pb "go.unistack.org/pkgdash/proto"
+)
+
+func init() {
+	storage.RegisterStorage("sqlite", NewStorage)
+}
+
+var _ storage.Storage = (*Sqlite)(nil)
+
+type Sqlite struct {
+	logger logger.Logger
+	db     *sqlx.DB
+}
+
+func NewStorage(log logger.Logger, db *sqlx.DB) interface{} {
+	return &Sqlite{db: db, logger: log}
+}
+
+func (s *Sqlite) PackageModulesCreate(ctx context.Context, pkg *models.Package, modules []*models.Module) error {
+	tx, err := s.db.BeginTxx(ctx, nil)
+	if err != nil {
+		return err
+	}
+
+	for _, mod := range modules {
+		err = tx.GetContext(ctx, mod, queryModulesCreate, mod.Name, mod.Version)
+		if err != nil {
+			_ = tx.Rollback()
+			return err
+		}
+		_, err = tx.ExecContext(ctx, queryPackagesModulesCreate, pkg.ID, mod.ID)
+		if err != nil {
+			_ = tx.Rollback()
+			return err
+		}
+
+	}
+
+	_, err = tx.ExecContext(ctx, queryPackageModulesCount, pkg.ID, len(modules))
+	if err != nil {
+		_ = tx.Rollback()
+		return err
+	}
+
+	if err = tx.Commit(); err != nil {
+		_ = tx.Rollback()
+		return err
+	}
+
+	return nil
+}
+
+func (s *Sqlite) PackageDelete(ctx context.Context, req *pb.PackageDeleteReq) error {
+	return fmt.Errorf("need implement")
+}
+
+func (s *Sqlite) PackageUpdate(ctx context.Context, req *pb.PackageUpdateReq) (*models.Package, error) {
+	return nil, fmt.Errorf("need implement")
+}
+
+func (s *Sqlite) PackageLookup(ctx context.Context, req *pb.PackageLookupReq) (*models.Package, error) {
+	pkg := &models.Package{}
+
+	err := s.db.GetContext(ctx, pkg, queryPackagesLookup, req.Id)
+	if err != nil {
+		if err == sql.ErrNoRows {
+			return nil, nil
+		}
+		return nil, err
+	}
+
+	return pkg, err
+}
+
+func (s *Sqlite) PackageList(ctx context.Context, req *pb.PackageListReq) ([]*models.Package, error) {
+	var packages []*models.Package
+
+	err := s.db.SelectContext(ctx, &packages, queryPackagesList)
+	if err != nil {
+		if err == sql.ErrNoRows {
+			return nil, nil
+		}
+		return nil, err
+	}
+
+	return packages, nil
+}
+
+func (s *Sqlite) PackageModules(ctx context.Context, req *pb.PackageModulesReq) ([]*models.Module, error) {
+	var modules []*models.Module
+
+	err := s.db.SelectContext(ctx, &modules, queryPackagesModules, req.Package)
+	if err != nil {
+		if err == sql.ErrNoRows {
+			return nil, nil
+		}
+		return nil, err
+	}
+
+	return modules, nil
+}
+
+func (s *Sqlite) CommentDelete(ctx context.Context, req *pb.CommentDeleteReq) error {
+	return nil
+}
+
+func (s *Sqlite) CommentCreate(ctx context.Context, req *pb.CommentCreateReq) (*models.Comment, error) {
+	tx, err := s.db.BeginTx(ctx, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	defer func() {
+		if err != nil {
+			if rollbackErr := tx.Rollback(); rollbackErr != nil {
+				s.logger.Error(ctx, "AddComment: unable to rollback: %v", rollbackErr)
+			}
+		} else {
+			err = tx.Commit()
+		}
+	}()
+
+	if _, err = tx.ExecContext(ctx, queryCommentsCreate, req.Comment, req.PackageId); err != nil {
+		return nil, err
+	}
+
+	return nil, nil
+}
+
+func (s *Sqlite) PackagesProcess(ctx context.Context, td time.Duration) ([]*models.Package, error) {
+	var packages []*models.Package
+	err := s.db.SelectContext(ctx, &packages, queryPackagesProcess, td.Seconds())
+	if err != nil {
+		return nil, err
+	}
+
+	return packages, nil
+}
+
+func (s *Sqlite) PackagesUpdateLastCheck(ctx context.Context, packages []*models.Package) error {
+	tx, err := s.db.BeginTxx(ctx, nil)
+	if err != nil {
+		return err
+	}
+	for _, pkg := range packages {
+		if _, err = tx.ExecContext(ctx, queryPackagesUpdateLastCheck, pkg.ID); err != nil {
+			tx.Rollback()
+			return err
+		}
+	}
+
+	if err = tx.Commit(); err != nil {
+		tx.Rollback()
+		return err
+	}
+
+	return nil
+}
+
+func (s *Sqlite) ModulesProcess(ctx context.Context, td time.Duration) ([]*models.Module, error) {
+	var modules []*models.Module
+	err := s.db.SelectContext(ctx, &modules, queryModulesProcess, td.Seconds())
+	if err != nil {
+		return nil, err
+	}
+
+	return modules, nil
+}
+
+func (s *Sqlite) PackageCreate(ctx context.Context, req *pb.PackageCreateReq) (*models.Package, error) {
+	pkg := &models.Package{}
+	err := s.db.GetContext(ctx, pkg, queryPackagesCreate, req.Name, req.Url)
+	if err != nil {
+		return nil, err
+	}
+
+	return pkg, nil
+}
+
+func (s *Sqlite) ModuleCreate(ctx context.Context, modules []*models.Module) error {
+	tx, err := s.db.BeginTxx(ctx, nil)
+	if err != nil {
+		return err
+	}
+
+	for _, mod := range modules {
+		err = tx.GetContext(ctx, mod, queryModulesCreate, mod.Name, mod.Version)
+		if err != nil {
+			_ = tx.Rollback()
+			return err
+		}
+	}
+
+	if err = tx.Commit(); err != nil {
+		_ = tx.Rollback()
+		return err
+	}
+
+	return nil
+}
+
+func (s *Sqlite) ModuleList(ctx context.Context, req *pb.ModuleListReq) ([]*models.Module, error) {
+	var modules []*models.Module
+
+	err := s.db.SelectContext(ctx, &modules, queryModulesList)
+	if err != nil {
+		if err == sql.ErrNoRows {
+			return nil, nil
+		}
+		return nil, err
+	}
+
+	return modules, nil
+}
+
+func (s *Sqlite) CommentList(ctx context.Context, req *pb.CommentListReq) ([]*models.Comment, error) {
+	var comments []*models.Comment
+
+	err := s.db.SelectContext(ctx, &comments, queryCommentsList, req.Package)
+	if err != nil {
+		if err == sql.ErrNoRows {
+			return nil, nil
+		}
+		return nil, err
+	}
+
+	return comments, nil
+}
+
+func (s *Sqlite) HandlerList(ctx context.Context, req *pb.HandlerListReq) ([]*models.Handler, error) {
+	var handlers []*models.Handler
+
+	err := s.db.SelectContext(ctx, &handlers, queryHandlersList, req.Package)
+	if err != nil {
+		if err == sql.ErrNoRows {
+			return nil, nil
+		}
+		return nil, err
+	}
+
+	return handlers, nil
+}
--- a/internal/storage/storage.go
+++ b/internal/storage/storage.go
@@ -0,0 +1,50 @@
+package storage
+
+import (
+	"context"
+	"errors"
+	"time"
+
+	"github.com/jmoiron/sqlx"
+	"go.unistack.org/micro/v3/logger"
+	"go.unistack.org/pkgdash/internal/models"
+	pb "go.unistack.org/pkgdash/proto"
+)
+
+func RegisterStorage(name string, fn func(logger.Logger, *sqlx.DB) interface{}) {
+	storages[name] = fn
+}
+
+var storages = map[string]func(logger.Logger, *sqlx.DB) interface{}{}
+
+type Storage interface {
+	PackageModulesCreate(ctx context.Context, pkg *models.Package, modules []*models.Module) error
+	PackagesUpdateLastCheck(ctx context.Context, packages []*models.Package) error
+	PackageModules(ctx context.Context, req *pb.PackageModulesReq) ([]*models.Module, error)
+	ModulesProcess(ctx context.Context, td time.Duration) ([]*models.Module, error)
+	PackagesProcess(ctx context.Context, td time.Duration) ([]*models.Package, error)
+	PackageCreate(ctx context.Context, req *pb.PackageCreateReq) (*models.Package, error)
+	HandlerList(ctx context.Context, req *pb.HandlerListReq) ([]*models.Handler, error)
+	PackageList(ctx context.Context, req *pb.PackageListReq) ([]*models.Package, error)
+	PackageLookup(ctx context.Context, req *pb.PackageLookupReq) (*models.Package, error)
+	PackageUpdate(ctx context.Context, req *pb.PackageUpdateReq) (*models.Package, error)
+	PackageDelete(ctx context.Context, req *pb.PackageDeleteReq) error
+	CommentCreate(ctx context.Context, req *pb.CommentCreateReq) (*models.Comment, error)
+	CommentDelete(ctx context.Context, req *pb.CommentDeleteReq) error
+	CommentList(ctx context.Context, req *pb.CommentListReq) ([]*models.Comment, error)
+	ModuleList(ctx context.Context, req *pb.ModuleListReq) ([]*models.Module, error)
+	ModuleCreate(ctx context.Context, modules []*models.Module) error
+}
+
+func NewStorage(name string, log logger.Logger, db *sqlx.DB) (Storage, error) {
+	fn, ok := storages[name]
+	if !ok {
+		return nil, errors.New("incorrect name store")
+	}
+	store := fn(log, db)
+	database, ok := store.(Storage)
+	if !ok {
+		return nil, errors.New("dont implements interface Storage")
+	}
+	return database, nil
+}
--- a/internal/storage/storage_test.go
+++ b/internal/storage/storage_test.go
@@ -0,0 +1,69 @@
+package storage
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+	"testing"
+
+	"go.unistack.org/pkgdash/internal/storage/sqlite"
+	pb "go.unistack.org/pkgdash/proto"
+)
+
+func TestGetModule(t *testing.T) {
+	conn, err := sql.Open("sqlite3", "/Users/devstigneev_local/GolandProjects/unistack/pkgdash/identifier.sqlite")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer conn.Close()
+	if err = conn.Ping(); err != nil {
+		t.Fatal(err)
+	}
+
+	st := sqlite.NewStorage()
+	store := st(conn, fs)
+
+	s, ok := store.(Storage)
+	if !ok {
+		t.Fatal("dont implements interface Storage")
+	}
+
+	req := &pb.GetModuleReq{
+		Id: []uint64{1, 2, 3},
+	}
+
+	module, err := s.GetModule(context.Background(), req)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	fmt.Println(module)
+}
+
+func TestGetComment(t *testing.T) {
+	conn, err := sql.Open("sqlite3", "/Users/devstigneev_local/GolandProjects/unistack/pkgdash/identifier.sqlite")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer conn.Close()
+	if err = conn.Ping(); err != nil {
+		t.Fatal(err)
+	}
+
+	st := sqlite.NewStorage()
+	store := st(conn, fs)
+
+	s, ok := store.(Storage)
+	if !ok {
+		t.Fatal("dont implements interface Storage")
+	}
+
+	req := &pb.GetCommentsReq{
+		Id: []uint64{1, 2, 3, 15},
+	}
+	comments, err := s.GetComment(context.Background(), req)
+	if err != nil {
+		t.Fatal(err)
+	}
+	fmt.Println(comments.Decode())
+}