diff --git a/.gitignore b/.gitignore index b55a950..9e82b91 100644 --- a/.gitignore +++ b/.gitignore @@ -27,3 +27,4 @@ bin/ tmp/ /pkgdash +*.sqlite diff --git a/go.mod b/go.mod index 6875102..5cda315 100644 --- a/go.mod +++ b/go.mod @@ -9,6 +9,7 @@ require ( github.com/google/uuid v1.3.0 github.com/jackc/pgtype v1.14.0 github.com/lib/pq v1.10.2 + github.com/mattn/go-sqlite3 v1.14.16 github.com/pkg/errors v0.9.1 go.unistack.org/cms-service v0.0.1 go.unistack.org/micro-client-http/v3 v3.9.3 diff --git a/go.sum b/go.sum index 874027e..5931b1c 100644 --- a/go.sum +++ b/go.sum @@ -747,6 +747,7 @@ github.com/mattn/go-shellwords v1.0.3/go.mod h1:3xCvwCdWdlDJUrvuMn7Wuy9eWs4pE8vq github.com/mattn/go-sqlite3 v1.9.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/mattn/go-sqlite3 v1.14.6/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= github.com/mattn/go-sqlite3 v1.14.16 h1:yOQRA0RpS5PFz/oikGwBEqvAWhWg5ufRz4ETLjwpU1Y= +github.com/mattn/go-sqlite3 v1.14.16/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= github.com/miekg/pkcs11 v1.0.3/go.mod h1:XsNlhZGX73bx86s2hdc/FuaLm2CPZJemRLMA+WTFxgs= diff --git a/handler/handlers.go b/handler/handlers.go index 0ce644b..1021fdf 100644 --- a/handler/handlers.go +++ b/handler/handlers.go @@ -2,16 +2,17 @@ package handler import ( "context" + "io" + "net/http" + cmsstorage "go.unistack.org/cms-service/storage" "go.unistack.org/micro/v3" "go.unistack.org/micro/v3/errors" "go.unistack.org/unistack-org/pkgdash/config" pb "go.unistack.org/unistack-org/pkgdash/proto/go_generate" - "go.unistack.org/unistack-org/pkgdash/service/client_git" + cligit "go.unistack.org/unistack-org/pkgdash/service/client_git" "go.unistack.org/unistack-org/pkgdash/storage" "google.golang.org/protobuf/encoding/protojson" - "io" - "net/http" ) type Handler struct { @@ -22,7 +23,7 @@ type Handler struct { protojson.MarshalOptions protojson.UnmarshalOptions - git client_git.Client + git cligit.Client chanUrl chan *pb.AddPackageRsp } @@ -150,7 +151,7 @@ func (h *Handler) AddPackage(w http.ResponseWriter, r *http.Request) { logger.Debug(ctx, "Success finish addPackage") } -func NewHandler(svc micro.Service, w writer, client client_git.Client) *Handler { +func NewHandler(svc micro.Service, w writer, client cligit.Client) *Handler { h := &Handler{ svc: svc, writer: w, diff --git a/service/client_git/client_test.go b/service/client_git/client_test.go index f6b07c1..784685f 100644 --- a/service/client_git/client_test.go +++ b/service/client_git/client_test.go @@ -7,11 +7,12 @@ import ( pb "go.unistack.org/unistack-org/pkgdash/proto/go_generate" "go.unistack.org/unistack-org/pkgdash/storage" "go.unistack.org/unistack-org/pkgdash/storage/postgres" + "go.unistack.org/unistack-org/pkgdash/storage/sqlite" "google.golang.org/protobuf/types/known/wrapperspb" "testing" ) -func TestClient(t *testing.T) { +func TestClientPG(t *testing.T) { dsn := fmt.Sprintf("user=%s password=%s host=%s port=%s dbname=%s sslmode=disable", "test", "123", "localhost", "5432", "postgres") conn, err := sql.Open("postgres", dsn) if err != nil { @@ -47,3 +48,39 @@ func TestClient(t *testing.T) { <-cli.Done() } + +func TestClientLite(t *testing.T) { + conn, err := sql.Open("sqlite3", "../../identifier.sqlite") + if err != nil { + t.Fatal(err) + } + defer conn.Close() + if err = conn.Ping(); err != nil { + t.Fatal(err) + } + + st, err := sqlite.NewStorage(conn) + if err != nil { + t.Fatal(err) + } + + s, ok := st.(storage.Storage) + if !ok { + t.Fatal("typecast error") + } + + ctx, cancel := context.WithCancel(context.Background()) + _ = cancel + cli := NewClient(1) + + ch := cli.Run(ctx, s) + + data := &pb.AddPackageRsp{ + Name: wrapperspb.String("test"), + Url: wrapperspb.String("https://github.com/dantedenis/service_history.git"), + } + + ch <- data + + <-cli.Done() +} diff --git a/service/service.go b/service/service.go index a80be6d..6c9303c 100644 --- a/service/service.go +++ b/service/service.go @@ -56,26 +56,26 @@ func NewService(ctx context.Context) (micro.Service, error) { h := handler.NewHandler(svc, writer, client_git.NewClient(5)) - if err := svc.Init( + if err = svc.Init( micro.AfterStart(func(_ context.Context) error { return h.Init(svc.Options().Context) }), micro.BeforeStart(func(ctx context.Context) error { - if err := config.Load(ctx, cs, config.LoadOverride(true)); err != nil { + if err = config.Load(ctx, cs, config.LoadOverride(true)); err != nil { return err } - if err := config.Validate(ctx, cfg); err != nil { + if err = config.Validate(ctx, cfg); err != nil { return err } - if err := svc.Init( + if err = svc.Init( micro.Name(cfg.Service.Name), micro.Version(cfg.Service.Version), ); err != nil { return err } - if err := svc.Server("http").Init( + if err = svc.Server("http").Init( server.Address(cfg.App.Address), server.Name(cfg.Service.Name), server.Version(cfg.Service.Version), diff --git a/storage/migrations/sqlite/000001_init_schema.down.sql b/storage/migrations/sqlite/000001_init_schema.down.sql new file mode 100644 index 0000000..b9ce01f --- /dev/null +++ b/storage/migrations/sqlite/000001_init_schema.down.sql @@ -0,0 +1,5 @@ +drop table if exists dashboard ; +drop table if exists package ; +drop table if exists module ; +drop table if exists issue ; +drop table if exists comment; \ No newline at end of file diff --git a/storage/migrations/sqlite/000001_init_schema.up.sql b/storage/migrations/sqlite/000001_init_schema.up.sql new file mode 100644 index 0000000..8893211 --- /dev/null +++ b/storage/migrations/sqlite/000001_init_schema.up.sql @@ -0,0 +1,38 @@ +create table if not exists dashboard ( + id integer primary key autoincrement not null , + "uuid" uuid not null unique , + package integer[] default '{}' +); + +create table if not exists comment ( + id integer primary key autoincrement not null , + "text" text , + created timestamp not null default current_timestamp , + updated timestamp +); + +create table if not exists module ( + id integer primary key autoincrement not null , + name varchar not null , + version varchar not null , + last_version varchar not null +); + +create table if not exists issue ( + id serial not null unique primary key , + --package integer references package(id) , + modules integer[] default '[]', + status integer default 0 , + "desc" varchar +); + +create table if not exists package ( + id integer primary key autoincrement not null , + name varchar not null , + url varchar , + modules integer[] default '[]', + issues integer[] default '[]', + comments integer[] default '[]' +); + + diff --git a/storage/sqlite/quries.go b/storage/sqlite/quries.go new file mode 100644 index 0000000..df92c68 --- /dev/null +++ b/storage/sqlite/quries.go @@ -0,0 +1,28 @@ +package sqlite + +const ( + queryListPackage = ` +select + id, + name, + url, + comments + --modules, + --issues, + from package; +` + queryAddComment = ` +insert into comment(text) values ($1) ; +update package +set comments = json_insert(comments, '$[#]', ( select last_insert_rowid() as id from comment )) +where id = $2 ; +` + queryAddPackage = ` +insert into package(name, url, modules) values ($1, $2, $3); +` + queryInsMsgGetIDs = ` +insert into module(name, version, last_version) values +%s +returning id; +` +) diff --git a/storage/sqlite/storage.go b/storage/sqlite/storage.go new file mode 100644 index 0000000..3019b18 --- /dev/null +++ b/storage/sqlite/storage.go @@ -0,0 +1,239 @@ +package sqlite + +import ( + "context" + "database/sql" + "embed" + "errors" + "fmt" + "strings" + + _ "github.com/mattn/go-sqlite3" + + "github.com/golang-migrate/migrate/v4" + "github.com/golang-migrate/migrate/v4/database/sqlite" + "github.com/golang-migrate/migrate/v4/source/iofs" + "github.com/lib/pq" + "go.unistack.org/micro/v3/logger" + "go.unistack.org/unistack-org/pkgdash/config" + "go.unistack.org/unistack-org/pkgdash/models" + pb "go.unistack.org/unistack-org/pkgdash/proto/go_generate" +) + +const ( + pathMigration = `migrations/sqlite` +) + +type Postgres struct { + db *sql.DB + fs embed.FS +} + +func NewStorage(db *sql.DB) (interface{}, error) { + return &Postgres{db: db}, nil +} + +func NewStorageFS(fs embed.FS) func(*sql.DB) (interface{}, error) { + return func(db *sql.DB) (interface{}, error) { + return &Postgres{db: db, fs: fs}, nil + } +} + +func (s *Postgres) MigrateUp() error { + driver, err := sqlite.WithInstance(s.db, &sqlite.Config{ + MigrationsTable: sqlite.DefaultMigrationsTable, + DatabaseName: config.ServiceName, + }) + if err != nil { + return err + } + source, err := iofs.New(s.fs, pathMigration) + if err != nil { + return err + } + + // TODO: pass own logger + m, err := migrate.NewWithInstance("fs", source, config.ServiceName, driver) + if err != nil { + return err + } + + if err = m.Up(); err != nil && !errors.Is(err, migrate.ErrNoChange) { + return err + } + + return nil +} + +func (s *Postgres) MigrateDown() error { + driver, err := sqlite.WithInstance(s.db, &sqlite.Config{ + MigrationsTable: sqlite.DefaultMigrationsTable, + DatabaseName: config.ServiceName, + }) + if err != nil { + return err + } + source, err := iofs.New(s.fs, pathMigration) + if err != nil { + return err + } + + // TODO: pass own logger + m, err := migrate.NewWithInstance("fs", source, config.ServiceName, driver) + if err != nil { + return err + } + + if err = m.Down(); err != nil && !errors.Is(err, migrate.ErrNoChange) { + return err + } + + return nil +} + +func (s *Postgres) UpdatePackage(ctx context.Context, rsp *pb.UpdatePackageRsp) error { + panic("need implement") +} + +func (s *Postgres) ListPackage(ctx context.Context) (models.ListPackage, error) { + rows, err := s.db.QueryContext(ctx, queryListPackage) + if err != nil { + return nil, err + } + + defer func() { + if err = rows.Close(); err != nil { + return + } + err = rows.Err() + }() + + result := make([]*models.Package, 0) + for rows.Next() { + tmp := &models.Package{} + if err = rows.Scan( + &tmp.ID, + &tmp.Name, + &tmp.URL, + pq.Array(&tmp.Comments), + ); err != nil { + return nil, err + } + } + + return result, err +} + +func (s *Postgres) AddComment(ctx context.Context, rsp *pb.AddCommentRsp) error { + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return err + } + + defer func() { + if err != nil { + if rollbackErr := tx.Rollback(); rollbackErr != nil { + logger.Errorf(ctx, "AddComment: unable to rollback: %v", rollbackErr) + } + } else { + err = tx.Commit() + } + }() + + res, err := tx.ExecContext(ctx, queryAddComment, rsp.Text, rsp.IdPackage.Value) + if err != nil { + return err + } + + if aff, affErr := res.RowsAffected(); err != nil { + err = affErr + } else if aff == 0 { + err = errors.New("rows affected is 0") + } + + return err +} + +func (s *Postgres) AddPackage(ctx context.Context, rsp *pb.AddPackageRsp) error { + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return err + } + + defer func() { + if err != nil { + if rollbackErr := tx.Rollback(); rollbackErr != nil { + logger.Errorf(ctx, "AddPackage: unable to rollback: %v", rollbackErr) + } + } else { + err = tx.Commit() + } + }() + + res, err := tx.ExecContext(ctx, queryAddPackage, rsp.Name.Value, rsp.Url.Value, pq.Array(rsp.Modules)) + if err != nil { + return err + } + + if aff, affErr := res.RowsAffected(); err != nil { + err = affErr + } else if aff == 0 { + err = errors.New("rows affected is 0") + } + + return err +} + +func (s *Postgres) InsertButchModules(ctx context.Context, rsp []models.Module) ([]uint64, error) { + tx, err := s.db.BeginTx(ctx, nil) + if err != nil { + return nil, err + } + defer func() { + if err != nil { + if rollbackErr := tx.Rollback(); rollbackErr != nil { + logger.Errorf(ctx, "AddPackage: unable to rollback: %v", rollbackErr) + } + } else { + err = tx.Commit() + } + }() + + query := generateQuery(rsp) + + rows, err := tx.QueryContext(ctx, query) + if err != nil { + return nil, err + } + defer func() { + if err = rows.Close(); err != nil { + return + } + err = rows.Err() + }() + + result := make([]uint64, 0) + for rows.Next() { + tmp := uint64(0) + if err = rows.Scan(&tmp); err != nil { + return nil, err + } + + result = append(result, tmp) + } + + return result, err +} + +func generateQuery(rsp []models.Module) string { + const pattern = `%c('%s', '%s', '%s')` + build := strings.Builder{} + comma := ' ' + for i := range rsp { + str := fmt.Sprintf(pattern, comma, rsp[i].Name, rsp[i].Version, rsp[i].LastVersion) + build.WriteString(str) + comma = ',' + } + + return fmt.Sprintf(queryInsMsgGetIDs, build.String()) +} diff --git a/storage/storage.go b/storage/storage.go index 616cb8c..64df6c5 100644 --- a/storage/storage.go +++ b/storage/storage.go @@ -7,6 +7,7 @@ import ( "go.unistack.org/unistack-org/pkgdash/models" pb "go.unistack.org/unistack-org/pkgdash/proto/go_generate" "go.unistack.org/unistack-org/pkgdash/storage/postgres" + "go.unistack.org/unistack-org/pkgdash/storage/sqlite" cmsstorage "go.unistack.org/cms-service/storage" ) @@ -20,6 +21,7 @@ var ( func init() { storages.RegisterStorage("postgres", postgres.NewStorageFS(fs)) + storages.RegisterStorage("sqlite", sqlite.NewStorageFS(fs)) } type Storage interface {