Signed-off-by: Vasiliy Tolstov <v.tolstov@unistack.org>
This commit is contained in:
2023-08-16 13:17:42 +03:00
parent 0678e72908
commit 78f0ae14d7
70 changed files with 4159 additions and 2355 deletions

View File

@@ -0,0 +1 @@
drop table if exists dashboard, package, module, issue, comment;

View File

@@ -0,0 +1,40 @@
create table if not exists dashboard (
id serial not null unique primary key ,
"uuid" uuid not null unique default gen_random_uuid() ,
package integer[] default '{}'::integer[]
);
create table if not exists comment (
id serial not null unique primary key ,
"text" text ,
package integer not null,
created timestamp not null default current_timestamp ,
updated timestamp default current_timestamp
);
create table if not exists module (
id serial not null unique primary key ,
name varchar not null ,
version varchar not null ,
last_version varchar not null
);
create table if not exists issue (
id serial not null unique primary key ,
--package integer references package(id) ,
modules integer[] default '{}'::integer[],
status integer default 0 ,
"desc" varchar
);
create table if not exists package (
id serial not null unique primary key ,
name varchar not null ,
url varchar ,
modules integer[] default '{}'::integer[],
issues integer[] default '{}'::integer[],
comments integer[] default '{}'::integer[]
);
create unique index module_info on module(name, version);

View File

@@ -0,0 +1,5 @@
drop table if exists dashboard ;
drop table if exists package ;
drop table if exists module ;
drop table if exists issue ;
drop table if exists comment;

View File

@@ -0,0 +1,39 @@
create table if not exists dashboard (
id integer primary key autoincrement not null ,
"uuid" uuid not null unique ,
package integer[] default '{}'
);
create table if not exists comment (
id integer primary key autoincrement not null ,
"text" text ,
package integer not null,
created timestamp not null default current_timestamp ,
updated timestamp default current_timestamp
);
create table if not exists module (
id integer primary key autoincrement not null ,
name varchar not null ,
version varchar not null ,
last_version varchar not null
);
create table if not exists issue (
id serial not null unique primary key ,
--package integer references package(id) ,
modules integer[] default '[]',
status integer default 0 ,
"desc" varchar
);
create table if not exists package (
id integer primary key autoincrement not null ,
name varchar not null ,
url varchar ,
modules integer[] default '[]',
issues integer[] default '[]',
comments integer[] default '[]'
);

View File

@@ -0,0 +1,32 @@
package postgres
const (
queryListPackage = `
select
id,
name,
url,
comments
--modules,
--issues,
from package;
`
queryAddComment = `
with insert_comm as (
insert into comment(text) values ($1) returning id
)
update package set comments = array_append(comments, (select * from insert_comm)) where id=$2;
`
queryAddPackage = `
insert into package(name, url, modules) values ($1, $2, $3);
`
queryInsMsgGetIDs = `
insert into module(name, version, last_version) values
%s
returning id;
`
queryGetModule = `
select id, name, version, last_version from module
where id in %s ;
`
)

View File

@@ -0,0 +1,233 @@
package postgres
import (
"context"
"database/sql"
"embed"
"errors"
"fmt"
"strings"
"github.com/golang-migrate/migrate/v4"
mpgx "github.com/golang-migrate/migrate/v4/database/pgx"
"github.com/golang-migrate/migrate/v4/source/iofs"
"github.com/lib/pq"
"go.unistack.org/micro/v4/logger"
"git.unistack.org/unistack-org/pkgdash/internal/config"
"git.unistack.org/unistack-org/pkgdash/internal/models"
pb "git.unistack.org/unistack-org/pkgdash/proto"
)
const (
pathMigration = `migrations/postgres`
)
type Postgres struct {
db *sql.DB
fs embed.FS
}
func NewStorage() func(*sql.DB, embed.FS) interface{} {
return func(db *sql.DB, fs embed.FS) interface{} {
return &Postgres{db: db, fs: fs}
}
}
func (s *Postgres) MigrateUp() error {
driver, err := mpgx.WithInstance(s.db, &mpgx.Config{
MigrationsTable: mpgx.DefaultMigrationsTable,
DatabaseName: config.ServiceName,
})
if err != nil {
return err
}
source, err := iofs.New(s.fs, pathMigration)
if err != nil {
return err
}
// TODO: pass own logger
m, err := migrate.NewWithInstance("fs", source, config.ServiceName, driver)
if err != nil {
return err
}
if err = m.Up(); err != nil && !errors.Is(err, migrate.ErrNoChange) {
return err
}
return nil
}
func (s *Postgres) MigrateDown() error {
driver, err := mpgx.WithInstance(s.db, &mpgx.Config{
MigrationsTable: mpgx.DefaultMigrationsTable,
DatabaseName: config.ServiceName,
})
if err != nil {
return err
}
source, err := iofs.New(s.fs, pathMigration)
if err != nil {
return err
}
// TODO: pass own logger
m, err := migrate.NewWithInstance("fs", source, config.ServiceName, driver)
if err != nil {
return err
}
if err = m.Down(); err != nil && !errors.Is(err, migrate.ErrNoChange) {
return err
}
return nil
}
func (s *Postgres) PackagesUpdate(ctx context.Context, req *pb.PackagesUpdateReq) error {
panic("need implement")
}
func (s *Postgres) PackagesList(ctx context.Context, req *pb.PackagesListReq) (models.ListPackage, error) {
rows, err := s.db.QueryContext(ctx, queryListPackage)
if err != nil {
return nil, err
}
defer func() {
if err = rows.Close(); err != nil {
return
}
err = rows.Err()
}()
result := make([]*models.Package, 0)
for rows.Next() {
tmp := &models.Package{}
if err = rows.Scan(
&tmp.ID,
&tmp.Name,
&tmp.URL,
pq.Array(&tmp.Comments),
); err != nil {
return nil, err
}
}
return result, err
}
func (s *Postgres) CommentsCreate(ctx context.Context, req *pb.CommentsCreateReq) error {
tx, err := s.db.BeginTx(ctx, nil)
if err != nil {
return err
}
defer func() {
if err != nil {
if rollbackErr := tx.Rollback(); rollbackErr != nil {
logger.Errorf(ctx, "AddComment: unable to rollback: %v", rollbackErr)
}
} else {
err = tx.Commit()
}
}()
res, err := tx.ExecContext(ctx, queryAddComment, req.Text, req.PackageId)
if err != nil {
return err
}
if aff, affErr := res.RowsAffected(); err != nil {
err = affErr
} else if aff == 0 {
err = errors.New("rows affected is 0")
}
return err
}
func (s *Postgres) PackagesCreate(ctx context.Context, req *pb.PackagesCreateReq) error {
tx, err := s.db.BeginTx(ctx, nil)
if err != nil {
return err
}
defer func() {
if err != nil {
if rollbackErr := tx.Rollback(); rollbackErr != nil {
logger.Errorf(ctx, "AddPackage: unable to rollback: %v", rollbackErr)
}
} else {
err = tx.Commit()
}
}()
res, err := tx.ExecContext(ctx, queryAddPackage, req.Name, req.Url, pq.Array(req.Modules))
if err != nil {
return err
}
if aff, affErr := res.RowsAffected(); err != nil {
err = affErr
} else if aff == 0 {
err = errors.New("rows affected is 0")
}
return err
}
func (s *Postgres) InsertButchModules(ctx context.Context, req []models.Module) ([]uint64, error) {
tx, err := s.db.BeginTx(ctx, nil)
if err != nil {
return nil, err
}
defer func() {
if err != nil {
if rollbackErr := tx.Rollback(); rollbackErr != nil {
logger.Errorf(ctx, "AddPackage: unable to rollback: %v", rollbackErr)
}
} else {
err = tx.Commit()
}
}()
query := generateQuery(req)
rows, err := tx.QueryContext(ctx, query)
if err != nil {
return nil, err
}
defer func() {
if err = rows.Close(); err != nil {
return
}
err = rows.Err()
}()
result := make([]uint64, 0)
for rows.Next() {
tmp := uint64(0)
if err = rows.Scan(&tmp); err != nil {
return nil, err
}
result = append(result, tmp)
}
return result, err
}
func generateQuery(rsp []models.Module) string {
const pattern = `%c('%s', '%s', '%s')`
build := strings.Builder{}
comma := ' '
for i := range rsp {
str := fmt.Sprintf(pattern, comma, rsp[i].Name, rsp[i].Version, rsp[i].LastVersion)
build.WriteString(str)
comma = ','
}
return fmt.Sprintf(queryInsMsgGetIDs, build.String())
}

View File

@@ -0,0 +1,38 @@
package postgres
import (
"fmt"
"testing"
"git.unistack.org/unistack-org/pkgdash/internal/models"
)
func TestGenerate(t *testing.T) {
m := []models.Module{
{
ID: 1,
Name: "test",
Version: "1.2.3",
Package: 2,
LastVersion: "23.31",
},
{
ID: 1,
Name: "321test",
Version: "1.3",
Package: 4,
LastVersion: "2111.31",
},
{
ID: 1,
Name: "testabcd",
Version: "1.2.3",
Package: 2,
LastVersion: "153453.31",
},
}
str := generateQuery(m)
fmt.Println(str)
}

View File

@@ -0,0 +1,34 @@
package sqlite
const (
queryPackagesList = `select
id,
name,
url,
comments
modules,
issues,
from package;
`
queryCommentsCreate = `
insert into comment(text) values ($1) returning id;
update package
set comments = json_insert(comments, '$[#]', ( select last_insert_rowid() as id from comment ))
where id = $2 ;
`
queryPackagesCreate = `
insert into package(name, url, modules) values ($1, $2, $3);
`
queryInsMsgGetIDs = `
insert into module(name, version, last_version) values
%s
returning id;
`
queryModulesList = `
select id, name, version, last_version, created, updated from modules;
`
queryCommentsList = `
select id, text, created, updated from comments;
`
)

View File

@@ -0,0 +1,321 @@
package sqlite
import (
"context"
"database/sql"
"embed"
"errors"
"fmt"
"strings"
"github.com/golang-migrate/migrate/v4"
"github.com/golang-migrate/migrate/v4/database/sqlite"
"github.com/golang-migrate/migrate/v4/source/iofs"
"github.com/lib/pq"
_ "github.com/mattn/go-sqlite3"
"go.unistack.org/micro/v4/logger"
"git.unistack.org/unistack-org/pkgdash/internal/models"
pb "git.unistack.org/unistack-org/pkgdash/proto"
)
const (
pathMigration = `migrations/sqlite`
)
type Sqlite struct {
db *sql.DB
fs embed.FS
}
func NewStorage() func(*sql.DB, embed.FS) interface{} {
return func(db *sql.DB, fs embed.FS) interface{} {
return &Sqlite{db: db, fs: fs}
}
}
func (s *Sqlite) MigrateUp() error {
driver, err := sqlite.WithInstance(s.db, &sqlite.Config{
MigrationsTable: sqlite.DefaultMigrationsTable,
DatabaseName: "pkgdash",
})
if err != nil {
return err
}
source, err := iofs.New(s.fs, pathMigration)
if err != nil {
return err
}
// TODO: pass own logger
m, err := migrate.NewWithInstance("fs", source, "pkgdash", driver)
if err != nil {
return err
}
if err = m.Up(); err != nil && !errors.Is(err, migrate.ErrNoChange) {
return err
}
return nil
}
func (s *Sqlite) MigrateDown() error {
driver, err := sqlite.WithInstance(s.db, &sqlite.Config{
MigrationsTable: sqlite.DefaultMigrationsTable,
DatabaseName: "pkgdash",
})
if err != nil {
return err
}
source, err := iofs.New(s.fs, pathMigration)
if err != nil {
return err
}
// TODO: pass own logger
m, err := migrate.NewWithInstance("fs", source, "pkgdash", driver)
if err != nil {
return err
}
if err = m.Down(); err != nil && !errors.Is(err, migrate.ErrNoChange) {
return err
}
return nil
}
func (s *Sqlite) PackagesUpdate(ctx context.Context, req *pb.PackagesUpdateReq) error {
panic("need implement")
}
func (s *Sqlite) PackagesList(ctx context.Context, req *pb.PackagesListReq) ([]*models.Package, error) {
var packages []*models.Package
rows, err := s.db.QueryContext(ctx, queryPackagesList)
if err != nil {
return nil, err
}
for ; rows.Err() == nil; rows.Next() {
pkg := &models.Package{}
if err = rows.Scan(
&pkg.ID,
&pkg.Name,
&pkg.URL,
&pkg.Comments,
); err != nil {
_ = rows.Close()
return nil, err
}
packages = append(packages, pkg)
}
if err = rows.Err(); err != nil {
return nil, err
}
if err = rows.Close(); err != nil {
return nil, err
}
return packages, err
}
func (s *Sqlite) CommentsCreate(ctx context.Context, req *pb.CommentsCreateReq) (id uint64, err error) {
tx, err := s.db.BeginTx(ctx, nil)
if err != nil {
return 0, err
}
defer func() {
if err != nil {
if rollbackErr := tx.Rollback(); rollbackErr != nil {
logger.Errorf(ctx, "AddComment: unable to rollback: %v", rollbackErr)
}
} else {
err = tx.Commit()
}
}()
if err = tx.QueryRowContext(ctx, queryCommentsCreate, req.Text, req.PackageId).Scan(&id); err != nil {
return id, err
}
return id, err
}
func (s *Sqlite) PackagesCreate(ctx context.Context, req *pb.PackagesCreateReq) error {
tx, err := s.db.BeginTx(ctx, nil)
if err != nil {
return err
}
defer func() {
if err != nil {
if rollbackErr := tx.Rollback(); rollbackErr != nil {
logger.Errorf(ctx, "AddPackage: unable to rollback: %v", rollbackErr)
}
} else {
err = tx.Commit()
}
}()
res, err := tx.ExecContext(ctx, queryPackagesCreate, req.Name, req.Url, pq.Array(req.Modules))
if err != nil {
return err
}
if aff, affErr := res.RowsAffected(); err != nil {
err = affErr
} else if aff == 0 {
err = errors.New("rows affected is 0")
}
return err
}
func (s *Sqlite) InsertButchModules(ctx context.Context, req []models.Module) ([]uint64, error) {
tx, err := s.db.BeginTx(ctx, nil)
if err != nil {
return nil, err
}
defer func() {
if err != nil {
if rollbackErr := tx.Rollback(); rollbackErr != nil {
logger.Errorf(ctx, "AddPackage: unable to rollback: %v", rollbackErr)
}
} else {
err = tx.Commit()
}
}()
query := generateQuery(req)
rows, err := tx.QueryContext(ctx, query)
if err != nil {
return nil, err
}
defer func() {
if err = rows.Close(); err != nil {
return
}
err = rows.Err()
}()
result := make([]uint64, 0)
for rows.Next() {
tmp := uint64(0)
if err = rows.Scan(&tmp); err != nil {
return nil, err
}
result = append(result, tmp)
}
return result, err
}
func (s *Sqlite) GetModule(ctx context.Context, req *pb.ModulesListReq) ([]*models.Module, error) {
var err error
var modules []*models.Module
rows, err := s.db.QueryContext(ctx, queryModulesList)
if err != nil {
return nil, err
}
defer func() {
if err = rows.Close(); err != nil {
return
}
err = rows.Err()
}()
for ; rows.Err() == nil; rows.Next() {
mod := &models.Module{}
if err = rows.Scan(
&mod.ID,
&mod.Name,
&mod.Version,
&mod.LastVersion,
); err != nil {
return nil, err
}
modules = append(modules, mod)
}
if err = rows.Err(); err != nil {
return nil, err
}
if err = rows.Close(); err != nil {
return nil, err
}
return modules, nil
}
func (s *Sqlite) CommentsList(ctx context.Context, req *pb.CommentsListReq) ([]*models.Comment, error) {
var comments []*models.Comment
rows, err := s.db.QueryContext(ctx, queryCommentsList, req.PackageId)
if err != nil {
return nil, err
}
defer func() {
if err = rows.Close(); err != nil {
return
}
err = rows.Err()
}()
for ; rows.Err() == nil; rows.Next() {
com := &models.Comment{}
if err = rows.Scan(
&com.ID,
&com.Text,
&com.Created,
&com.Updated,
); err != nil {
_ = rows.Close()
return nil, err
}
comments = append(comments, com)
}
if err = rows.Err(); err != nil {
return nil, err
}
if err = rows.Close(); err != nil {
return nil, err
}
return comments, nil
}
func convertSliceUInt(arg ...uint64) []interface{} {
result := make([]interface{}, 0, len(arg))
for i := range arg {
result = append(result, arg[i])
}
return result
}
func generateQuery(rsp []models.Module) string {
const pattern = `%c('%s', '%s', '%s')`
build := strings.Builder{}
comma := ' '
for i := range rsp {
str := fmt.Sprintf(pattern, comma, rsp[i].Name, rsp[i].Version, rsp[i].LastVersion)
build.WriteString(str)
comma = ','
}
return fmt.Sprintf(queryInsMsgGetIDs, build.String())
}
func generateArrayIneq(count int) string {
return "(?" + strings.Repeat(",?", count-1) + ")"
}

View File

@@ -0,0 +1,69 @@
package storage
import (
"context"
"database/sql"
"embed"
"errors"
"git.unistack.org/unistack-org/pkgdash/internal/models"
// "git.unistack.org/unistack-org/pkgdash/internal/storage/postgres"
"git.unistack.org/unistack-org/pkgdash/internal/storage/sqlite"
pb "git.unistack.org/unistack-org/pkgdash/proto"
)
//go:embed migrations
var fs embed.FS
var storages = map[string]func(*sql.DB, embed.FS) interface{}{
//"postgres": postgres.NewStorage(),
"sqlite": sqlite.NewStorage(),
}
type contextKey string
var storeIdent = contextKey("store")
type Migrate interface {
MigrateUp() error
MigrateDown() error
}
type Storage interface {
Migrate
PackagesCreate(ctx context.Context, req *pb.PackagesCreateReq) error
PackagesList(ctx context.Context, req *pb.PackagesListReq) ([]*models.Package, error)
PackagesUpdate(ctx context.Context, req *pb.PackagesUpdateReq) error
PackagesDelete(ctx context.Context, req *pb.PackagesDeleteReq) error
CommentsCreate(ctx context.Context, req *pb.CommentsCreateReq) (*models.Comment, error)
CommentsDelete(ctx context.Context, req *pb.CommentsDeleteReq) error
CommentsList(ctx context.Context, req *pb.CommentsListReq) ([]*models.Comment, error)
InsertButchModules(ctx context.Context, req []models.Module) ([]uint64, error)
ModulesList(ctx context.Context, req *pb.ModulesListReq) ([]*models.Module, error)
}
func NewStorage(name string, db *sql.DB) (Storage, error) {
function, ok := storages[name]
if !ok {
return nil, errors.New("incorrect name store")
}
store := function(db, fs)
database, ok := store.(Storage)
if !ok {
return nil, errors.New("dont implements interface Storage")
}
return database, nil
}
func InContext(ctx context.Context, val Storage) context.Context {
return context.WithValue(ctx, storeIdent, val)
}
func FromContext(ctx context.Context) (Storage, error) {
if store, ok := ctx.Value(storeIdent).(Storage); !ok {
return nil, errors.New("empty store")
} else {
return store, nil
}
}

View File

@@ -0,0 +1,69 @@
package storage
import (
"context"
"database/sql"
"fmt"
"testing"
"git.unistack.org/unistack-org/pkgdash/internal/storage/sqlite"
pb "git.unistack.org/unistack-org/pkgdash/proto"
)
func TestGetModule(t *testing.T) {
conn, err := sql.Open("sqlite3", "/Users/devstigneev_local/GolandProjects/unistack/pkgdash/identifier.sqlite")
if err != nil {
t.Fatal(err)
}
defer conn.Close()
if err = conn.Ping(); err != nil {
t.Fatal(err)
}
st := sqlite.NewStorage()
store := st(conn, fs)
s, ok := store.(Storage)
if !ok {
t.Fatal("dont implements interface Storage")
}
req := &pb.GetModuleReq{
Id: []uint64{1, 2, 3},
}
module, err := s.GetModule(context.Background(), req)
if err != nil {
t.Fatal(err)
}
fmt.Println(module)
}
func TestGetComment(t *testing.T) {
conn, err := sql.Open("sqlite3", "/Users/devstigneev_local/GolandProjects/unistack/pkgdash/identifier.sqlite")
if err != nil {
t.Fatal(err)
}
defer conn.Close()
if err = conn.Ping(); err != nil {
t.Fatal(err)
}
st := sqlite.NewStorage()
store := st(conn, fs)
s, ok := store.(Storage)
if !ok {
t.Fatal("dont implements interface Storage")
}
req := &pb.GetCommentsReq{
Id: []uint64{1, 2, 3, 15},
}
comments, err := s.GetComment(context.Background(), req)
if err != nil {
t.Fatal(err)
}
fmt.Println(comments.Decode())
}