add handlers, storage(Postgres, sqlite) (#3)

Reviewed-on: #3
Co-authored-by: Evstigneev Denis <danteevstigneev@yandex.ru>
Co-committed-by: Evstigneev Denis <danteevstigneev@yandex.ru>

storage/migrations/postgres/000001_init_schema.up.sql

@@ -1,12 +1,12 @@
 create table if not exists dashboard (
     id serial not null unique primary key ,
-    "uniq_id" uuid not null unique default gen_random_uuid() ,
-    package integer
+    "uuid" uuid not null unique default gen_random_uuid() ,
+    package integer[] default '{}'::integer[]
 );
 
 create table if not exists comment (
     id serial not null unique primary key ,
-    text text ,
+    "text" text ,
     created timestamp not null default current_timestamp ,
     updated timestamp
 );
@@ -14,13 +14,14 @@ create table if not exists comment (
 create table if not exists module (
     id serial not null unique primary key ,
     name varchar not null ,
-    version varchar not null
+    version varchar not null ,
+    last_version varchar not null
 );
 
 create table if not exists issue (
     id serial not null unique primary key ,
     --package integer references package(id) ,
-    modules integer[] ,
+    modules integer[] default '{}'::integer[],
     status integer default 0 ,
     "desc" varchar
 );
@@ -29,9 +30,10 @@ create table if not exists package (
     id serial not null unique primary key ,
     name varchar not null ,
     url varchar ,
-    modules integer[] ,
-    issues integer[] ,
-    comments integer[]
+    modules integer[] default '{}'::integer[],
+    issues integer[] default '{}'::integer[],
+    comments integer[] default '{}'::integer[]
 );
 
+create unique index module_info on module(name, version);
 

storage/migrations/sqlite/000001_init_schema.down.sql

@@ -0,0 +1,5 @@
+drop table if exists dashboard ;
+drop table if exists package ;
+drop table if exists module ;
+drop table if exists issue ;
+drop table if exists comment;

storage/migrations/sqlite/000001_init_schema.up.sql

@@ -0,0 +1,38 @@
+create table if not exists dashboard (
+    id integer primary key autoincrement not null ,
+    "uuid" uuid not null unique ,
+    package integer[] default '{}'
+);
+
+create table if not exists comment (
+    id integer primary key autoincrement not null ,
+    "text" text ,
+    created timestamp not null default current_timestamp ,
+    updated timestamp
+);
+
+create table if not exists module (
+    id integer primary key autoincrement not null ,
+    name varchar not null ,
+    version varchar not null ,
+    last_version varchar not null
+);
+
+create table if not exists issue (
+    id serial not null unique primary key ,
+    --package integer references package(id) ,
+    modules integer[] default '[]',
+    status integer default 0 ,
+    "desc" varchar
+);
+
+create table if not exists package (
+    id integer primary key autoincrement not null ,
+    name varchar not null ,
+    url varchar ,
+    modules integer[] default '[]',
+    issues integer[] default '[]',
+    comments integer[] default '[]'
+);
+
+

storage/postgres/quries.go

@@ -2,6 +2,27 @@ package postgres
 
 const (
 	queryListPackage = `
-select * from package;
+select 
+    id,
+    name,
+    url,
+	comments
+	--modules,
+	--issues,
+    from package;
+`
+	queryAddComment = `
+with insert_comm as (
+    insert into comment(text) values ($1) returning id
+)
+update package set comments = array_append(comments, (select * from insert_comm)) where id=$2;
+`
+	queryAddPackage = `
+insert into package(name, url, modules) values ($1, $2, $3);
+`
+	queryInsMsgGetIDs = `
+insert into module(name, version, last_version)  values
+%s
+returning id;
 `
 )

storage/postgres/storage.go

@@ -1,14 +1,21 @@
 package postgres
 
 import (
+	"context"
 	"database/sql"
 	"embed"
 	"errors"
-	"go.unistack.org/unistack-org/pkgdash/config"
+	"fmt"
+	pb "go.unistack.org/unistack-org/pkgdash/proto/go_generate"
+	"strings"
 
 	"github.com/golang-migrate/migrate/v4"
-	"github.com/golang-migrate/migrate/v4/database/pgx"
+	mpgx "github.com/golang-migrate/migrate/v4/database/pgx"
 	"github.com/golang-migrate/migrate/v4/source/iofs"
+	"github.com/lib/pq"
+	"go.unistack.org/micro/v3/logger"
+	"go.unistack.org/unistack-org/pkgdash/config"
+	"go.unistack.org/unistack-org/pkgdash/models"
 )
 
 const (
@@ -31,8 +38,8 @@ func NewStorageFS(fs embed.FS) func(*sql.DB) (interface{}, error) {
 }
 
 func (s *Postgres) MigrateUp() error {
-	driver, err := pgx.WithInstance(s.db, &pgx.Config{
-		MigrationsTable: pgx.DefaultMigrationsTable,
+	driver, err := mpgx.WithInstance(s.db, &mpgx.Config{
+		MigrationsTable: mpgx.DefaultMigrationsTable,
 		DatabaseName:    config.ServiceName,
 	})
 	if err != nil {
@@ -57,8 +64,8 @@ func (s *Postgres) MigrateUp() error {
 }
 
 func (s *Postgres) MigrateDown() error {
-	driver, err := pgx.WithInstance(s.db, &pgx.Config{
-		MigrationsTable: pgx.DefaultMigrationsTable,
+	driver, err := mpgx.WithInstance(s.db, &mpgx.Config{
+		MigrationsTable: mpgx.DefaultMigrationsTable,
 		DatabaseName:    config.ServiceName,
 	})
 	if err != nil {
@@ -81,3 +88,150 @@ func (s *Postgres) MigrateDown() error {
 
 	return nil
 }
+
+func (s *Postgres) UpdatePackage(ctx context.Context, req *pb.UpdatePackageReq) error {
+	panic("need implement")
+}
+
+func (s *Postgres) ListPackage(ctx context.Context) (models.ListPackage, error) {
+	rows, err := s.db.QueryContext(ctx, queryListPackage)
+	if err != nil {
+		return nil, err
+	}
+
+	defer func() {
+		if err = rows.Close(); err != nil {
+			return
+		}
+		err = rows.Err()
+	}()
+
+	result := make([]*models.Package, 0)
+	for rows.Next() {
+		tmp := &models.Package{}
+		if err = rows.Scan(
+			&tmp.ID,
+			&tmp.Name,
+			&tmp.URL,
+			pq.Array(&tmp.Comments),
+		); err != nil {
+			return nil, err
+		}
+	}
+
+	return result, err
+}
+
+func (s *Postgres) AddComment(ctx context.Context, req *pb.AddCommentReq) error {
+	tx, err := s.db.BeginTx(ctx, nil)
+	if err != nil {
+		return err
+	}
+
+	defer func() {
+		if err != nil {
+			if rollbackErr := tx.Rollback(); rollbackErr != nil {
+				logger.Errorf(ctx, "AddComment: unable to rollback: %v", rollbackErr)
+			}
+		} else {
+			err = tx.Commit()
+		}
+	}()
+
+	res, err := tx.ExecContext(ctx, queryAddComment, req.Text, req.IdPackage.Value)
+	if err != nil {
+		return err
+	}
+
+	if aff, affErr := res.RowsAffected(); err != nil {
+		err = affErr
+	} else if aff == 0 {
+		err = errors.New("rows affected is 0")
+	}
+
+	return err
+}
+
+func (s *Postgres) AddPackage(ctx context.Context, req *pb.AddPackageReq) error {
+	tx, err := s.db.BeginTx(ctx, nil)
+	if err != nil {
+		return err
+	}
+
+	defer func() {
+		if err != nil {
+			if rollbackErr := tx.Rollback(); rollbackErr != nil {
+				logger.Errorf(ctx, "AddPackage: unable to rollback: %v", rollbackErr)
+			}
+		} else {
+			err = tx.Commit()
+		}
+	}()
+
+	res, err := tx.ExecContext(ctx, queryAddPackage, req.Name.Value, req.Url.Value, pq.Array(req.Modules))
+	if err != nil {
+		return err
+	}
+
+	if aff, affErr := res.RowsAffected(); err != nil {
+		err = affErr
+	} else if aff == 0 {
+		err = errors.New("rows affected is 0")
+	}
+
+	return err
+}
+
+func (s *Postgres) InsertButchModules(ctx context.Context, req []models.Module) ([]uint64, error) {
+	tx, err := s.db.BeginTx(ctx, nil)
+	if err != nil {
+		return nil, err
+	}
+	defer func() {
+		if err != nil {
+			if rollbackErr := tx.Rollback(); rollbackErr != nil {
+				logger.Errorf(ctx, "AddPackage: unable to rollback: %v", rollbackErr)
+			}
+		} else {
+			err = tx.Commit()
+		}
+	}()
+
+	query := generateQuery(req)
+
+	rows, err := tx.QueryContext(ctx, query)
+	if err != nil {
+		return nil, err
+	}
+	defer func() {
+		if err = rows.Close(); err != nil {
+			return
+		}
+		err = rows.Err()
+	}()
+
+	result := make([]uint64, 0)
+	for rows.Next() {
+		tmp := uint64(0)
+		if err = rows.Scan(&tmp); err != nil {
+			return nil, err
+		}
+
+		result = append(result, tmp)
+	}
+
+	return result, err
+}
+
+func generateQuery(rsp []models.Module) string {
+	const pattern = `%c('%s', '%s', '%s')`
+	build := strings.Builder{}
+	comma := ' '
+	for i := range rsp {
+		str := fmt.Sprintf(pattern, comma, rsp[i].Name, rsp[i].Version, rsp[i].LastVersion)
+		build.WriteString(str)
+		comma = ','
+	}
+
+	return fmt.Sprintf(queryInsMsgGetIDs, build.String())
+}

storage/postgres/storage_test.go

@@ -0,0 +1,37 @@
+package postgres
+
+import (
+	"fmt"
+	"go.unistack.org/unistack-org/pkgdash/models"
+	"testing"
+)
+
+func TestGenerate(t *testing.T) {
+	m := []models.Module{
+		{
+			ID:          1,
+			Name:        "test",
+			Version:     "1.2.3",
+			Package:     2,
+			LastVersion: "23.31",
+		},
+		{
+			ID:          1,
+			Name:        "321test",
+			Version:     "1.3",
+			Package:     4,
+			LastVersion: "2111.31",
+		},
+		{
+			ID:          1,
+			Name:        "testabcd",
+			Version:     "1.2.3",
+			Package:     2,
+			LastVersion: "153453.31",
+		},
+	}
+
+	str := generateQuery(m)
+
+	fmt.Println(str)
+}

storage/sqlite/quries.go

@@ -0,0 +1,32 @@
+package sqlite
+
+const (
+	queryListPackage = `
+select 
+    id,
+    name,
+    url,
+	comments
+	--modules,
+	--issues,
+    from package;
+`
+	queryAddComment = `
+insert into comment(text) values ($1) ;
+update package
+set comments = json_insert(comments, '$[#]', ( select last_insert_rowid() as id from comment ))
+where id = $2 ;
+`
+	queryAddPackage = `
+insert into package(name, url, modules) values ($1, $2, $3);
+`
+	queryInsMsgGetIDs = `
+insert into module(name, version, last_version)  values
+%s
+returning id;
+`
+	queryGetModule = `
+select id, name, version, last_version from module
+where id in %s ;
+`
+)

storage/sqlite/storage.go

@@ -0,0 +1,287 @@
+package sqlite
+
+import (
+	"context"
+	"database/sql"
+	"embed"
+	"errors"
+	"fmt"
+	"strings"
+
+	_ "github.com/mattn/go-sqlite3"
+
+	"github.com/golang-migrate/migrate/v4"
+	"github.com/golang-migrate/migrate/v4/database/sqlite"
+	"github.com/golang-migrate/migrate/v4/source/iofs"
+	"github.com/lib/pq"
+	"go.unistack.org/micro/v3/logger"
+	"go.unistack.org/unistack-org/pkgdash/config"
+	"go.unistack.org/unistack-org/pkgdash/models"
+	pb "go.unistack.org/unistack-org/pkgdash/proto/go_generate"
+)
+
+const (
+	pathMigration = `migrations/sqlite`
+)
+
+type Sqlite struct {
+	db *sql.DB
+	fs embed.FS
+}
+
+func NewStorage(db *sql.DB) (interface{}, error) {
+	return &Sqlite{db: db}, nil
+}
+
+func NewStorageFS(fs embed.FS) func(*sql.DB) (interface{}, error) {
+	return func(db *sql.DB) (interface{}, error) {
+		return &Sqlite{db: db, fs: fs}, nil
+	}
+}
+
+func (s *Sqlite) MigrateUp() error {
+	driver, err := sqlite.WithInstance(s.db, &sqlite.Config{
+		MigrationsTable: sqlite.DefaultMigrationsTable,
+		DatabaseName:    config.ServiceName,
+	})
+	if err != nil {
+		return err
+	}
+	source, err := iofs.New(s.fs, pathMigration)
+	if err != nil {
+		return err
+	}
+
+	// TODO: pass own logger
+	m, err := migrate.NewWithInstance("fs", source, config.ServiceName, driver)
+	if err != nil {
+		return err
+	}
+
+	if err = m.Up(); err != nil && !errors.Is(err, migrate.ErrNoChange) {
+		return err
+	}
+
+	return nil
+}
+
+func (s *Sqlite) MigrateDown() error {
+	driver, err := sqlite.WithInstance(s.db, &sqlite.Config{
+		MigrationsTable: sqlite.DefaultMigrationsTable,
+		DatabaseName:    config.ServiceName,
+	})
+	if err != nil {
+		return err
+	}
+	source, err := iofs.New(s.fs, pathMigration)
+	if err != nil {
+		return err
+	}
+
+	// TODO: pass own logger
+	m, err := migrate.NewWithInstance("fs", source, config.ServiceName, driver)
+	if err != nil {
+		return err
+	}
+
+	if err = m.Down(); err != nil && !errors.Is(err, migrate.ErrNoChange) {
+		return err
+	}
+
+	return nil
+}
+
+func (s *Sqlite) UpdatePackage(ctx context.Context, req *pb.UpdatePackageReq) error {
+	panic("need implement")
+}
+
+func (s *Sqlite) ListPackage(ctx context.Context) (models.ListPackage, error) {
+	rows, err := s.db.QueryContext(ctx, queryListPackage)
+	if err != nil {
+		return nil, err
+	}
+
+	defer func() {
+		if err = rows.Close(); err != nil {
+			return
+		}
+		err = rows.Err()
+	}()
+
+	result := make([]*models.Package, 0)
+	for rows.Next() {
+		tmp := &models.Package{}
+		if err = rows.Scan(
+			&tmp.ID,
+			&tmp.Name,
+			&tmp.URL,
+			pq.Array(&tmp.Comments),
+		); err != nil {
+			return nil, err
+		}
+	}
+
+	return result, err
+}
+
+func (s *Sqlite) AddComment(ctx context.Context, req *pb.AddCommentReq) error {
+	tx, err := s.db.BeginTx(ctx, nil)
+	if err != nil {
+		return err
+	}
+
+	defer func() {
+		if err != nil {
+			if rollbackErr := tx.Rollback(); rollbackErr != nil {
+				logger.Errorf(ctx, "AddComment: unable to rollback: %v", rollbackErr)
+			}
+		} else {
+			err = tx.Commit()
+		}
+	}()
+
+	res, err := tx.ExecContext(ctx, queryAddComment, req.Text, req.IdPackage.Value)
+	if err != nil {
+		return err
+	}
+
+	if aff, affErr := res.RowsAffected(); err != nil {
+		err = affErr
+	} else if aff == 0 {
+		err = errors.New("rows affected is 0")
+	}
+
+	return err
+}
+
+func (s *Sqlite) AddPackage(ctx context.Context, req *pb.AddPackageReq) error {
+	tx, err := s.db.BeginTx(ctx, nil)
+	if err != nil {
+		return err
+	}
+
+	defer func() {
+		if err != nil {
+			if rollbackErr := tx.Rollback(); rollbackErr != nil {
+				logger.Errorf(ctx, "AddPackage: unable to rollback: %v", rollbackErr)
+			}
+		} else {
+			err = tx.Commit()
+		}
+	}()
+
+	res, err := tx.ExecContext(ctx, queryAddPackage, req.Name.Value, req.Url.Value, pq.Array(req.Modules))
+	if err != nil {
+		return err
+	}
+
+	if aff, affErr := res.RowsAffected(); err != nil {
+		err = affErr
+	} else if aff == 0 {
+		err = errors.New("rows affected is 0")
+	}
+
+	return err
+}
+
+func (s *Sqlite) InsertButchModules(ctx context.Context, req []models.Module) ([]uint64, error) {
+	tx, err := s.db.BeginTx(ctx, nil)
+	if err != nil {
+		return nil, err
+	}
+	defer func() {
+		if err != nil {
+			if rollbackErr := tx.Rollback(); rollbackErr != nil {
+				logger.Errorf(ctx, "AddPackage: unable to rollback: %v", rollbackErr)
+			}
+		} else {
+			err = tx.Commit()
+		}
+	}()
+
+	query := generateQuery(req)
+
+	rows, err := tx.QueryContext(ctx, query)
+	if err != nil {
+		return nil, err
+	}
+	defer func() {
+		if err = rows.Close(); err != nil {
+			return
+		}
+		err = rows.Err()
+	}()
+
+	result := make([]uint64, 0)
+	for rows.Next() {
+		tmp := uint64(0)
+		if err = rows.Scan(&tmp); err != nil {
+			return nil, err
+		}
+
+		result = append(result, tmp)
+	}
+
+	return result, err
+}
+
+func (s *Sqlite) GetModule(ctx context.Context, req *pb.GetModuleReq) (result models.ListModule, err error) {
+	query := ""
+	if len(req.Id) < 1 {
+		query = fmt.Sprintf(queryGetModule, "() or 1=1")
+	} else {
+		query = fmt.Sprintf(queryGetModule, generateArrayIneq(len(req.Id)))
+	}
+
+	rows, err := s.db.QueryContext(ctx, query, convertSliceUInt(req.Id...)...)
+	if err != nil {
+		return nil, err
+	}
+	defer func() {
+		if err = rows.Close(); err != nil {
+			return
+		}
+		err = rows.Err()
+	}()
+
+	for rows.Next() {
+		tmp := &models.Module{}
+		if err = rows.Scan(
+			&tmp.ID,
+			&tmp.Name,
+			&tmp.Version,
+			&tmp.LastVersion,
+		); err != nil {
+			return nil, err
+		}
+
+		result = append(result, tmp)
+	}
+
+	return result, err
+}
+
+func convertSliceUInt(arg ...uint64) []interface{} {
+	result := make([]interface{}, 0, len(arg))
+	for i := range arg {
+		result = append(result, arg[i])
+	}
+	return result
+}
+
+func generateQuery(rsp []models.Module) string {
+	const pattern = `%c('%s', '%s', '%s')`
+	build := strings.Builder{}
+	comma := ' '
+	for i := range rsp {
+		str := fmt.Sprintf(pattern, comma, rsp[i].Name, rsp[i].Version, rsp[i].LastVersion)
+		build.WriteString(str)
+		comma = ','
+	}
+
+	return fmt.Sprintf(queryInsMsgGetIDs, build.String())
+}
+
+func generateArrayIneq(count int) string {
+	return "(?" + strings.Repeat(",?", count-1) + ")"
+}

storage/storage.go

@@ -5,7 +5,9 @@ import (
 	"database/sql"
 	"embed"
 	"go.unistack.org/unistack-org/pkgdash/models"
+	pb "go.unistack.org/unistack-org/pkgdash/proto/go_generate"
 	"go.unistack.org/unistack-org/pkgdash/storage/postgres"
+	"go.unistack.org/unistack-org/pkgdash/storage/sqlite"
 
 	cmsstorage "go.unistack.org/cms-service/storage"
 )
@@ -19,12 +21,18 @@ var (
 
 func init() {
 	storages.RegisterStorage("postgres", postgres.NewStorageFS(fs))
+	storages.RegisterStorage("sqlite", sqlite.NewStorageFS(fs))
 }
 
 type Storage interface {
 	cmsstorage.Migrator
 
-	List(ctx context.Context) ([]*models.Package, error)
+	ListPackage(ctx context.Context) (models.ListPackage, error)
+	UpdatePackage(ctx context.Context, req *pb.UpdatePackageReq) error
+	AddComment(ctx context.Context, req *pb.AddCommentReq) error
+	AddPackage(ctx context.Context, req *pb.AddPackageReq) error
+	InsertButchModules(ctx context.Context, req []models.Module) ([]uint64, error)
+	GetModule(ctx context.Context, req *pb.GetModuleReq) (models.ListModule, error)
 }
 
 func NewStorage(name string, db *sql.DB) (interface{}, error) {

storage/storage_test.go

@@ -0,0 +1,42 @@
+package storage
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+	pb "go.unistack.org/unistack-org/pkgdash/proto/go_generate"
+	"go.unistack.org/unistack-org/pkgdash/storage/sqlite"
+	"testing"
+)
+
+func TestGetModule(t *testing.T) {
+	conn, err := sql.Open("sqlite3", "/Users/devstigneev_local/GolandProjects/unistack/pkgdash/identifier.sqlite")
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer conn.Close()
+	if err = conn.Ping(); err != nil {
+		t.Fatal(err)
+	}
+
+	st, err := sqlite.NewStorage(conn)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	s, ok := st.(Storage)
+	if !ok {
+		t.Fatal("typecast error")
+	}
+
+	req := &pb.GetModuleReq{
+		Id: []uint64{1, 2, 5, 40},
+	}
+
+	module, err := s.GetModule(context.Background(), req)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	fmt.Println(module)
+}