diff --git a/util/test/test.go b/util/test/test.go new file mode 100644 index 00000000..eedc4e77 --- /dev/null +++ b/util/test/test.go @@ -0,0 +1,505 @@ +package test + +import ( + "bufio" + "bytes" + "context" + "database/sql/driver" + "encoding/csv" + "fmt" + "io" + "os" + "path" + "path/filepath" + "reflect" + "strings" + "time" + + sqlmock "github.com/DATA-DOG/go-sqlmock" + "go.unistack.org/micro/v3/client" + "go.unistack.org/micro/v3/codec" + "go.unistack.org/micro/v3/errors" + "go.unistack.org/micro/v3/metadata" + "golang.org/x/sync/errgroup" + "google.golang.org/grpc/status" + "google.golang.org/protobuf/proto" +) + +var ErrUnknownContentType = fmt.Errorf("unknown content type") + +type Extension struct { + Ext []string +} + +var ( + // ExtToTypes map file extension to content type + ExtToTypes = map[string][]string{ + "json": {"application/json", "application/grpc+json"}, + "yaml": {"application/yaml", "application/yml", "text/yaml", "text/yml"}, + "yml": {"application/yaml", "application/yml", "text/yaml", "text/yml"}, + "proto": {"application/grpc", "application/grpc+proto", "application/proto"}, + } + // DefaultExts specifies default file extensions to load data + DefaultExts = []string{"csv", "json", "yaml", "yml", "proto"} + // Codecs map to detect codec for test file or request content type + Codecs map[string]codec.Codec + + // ResponseCompareFunc used to compare actual response with test case data + ResponseCompareFunc = func(expectRsp []byte, testRsp interface{}, expectCodec codec.Codec, testCodec codec.Codec) error { + var err error + + expectMap := make(map[string]interface{}) + if err = expectCodec.Unmarshal(expectRsp, &expectMap); err != nil { + return fmt.Errorf("failed to unmarshal err: %w", err) + } + + testMap := make(map[string]interface{}) + switch v := testRsp.(type) { + case *codec.Frame: + if err = testCodec.Unmarshal(v.Data, &testMap); err != nil { + return fmt.Errorf("failed to unmarshal err: %w", err) + } + case *errors.Error: + if err = expectCodec.Unmarshal([]byte(v.Error()), &testMap); err != nil { + return fmt.Errorf("failed to unmarshal err: %w", err) + } + case error: + st, ok := status.FromError(v) + if !ok { + return v + } + me := errors.Parse(st.Message()) + if me.Code != 0 { + if err = expectCodec.Unmarshal([]byte(me.Error()), &testMap); err != nil { + return fmt.Errorf("failed to unmarshal err: %w", err) + } + break + } + for _, se := range st.Details() { + switch ne := se.(type) { + case proto.Message: + buf, err := testCodec.Marshal(ne) + if err != nil { + return fmt.Errorf("failed to marshal err: %w", err) + } + if err = testCodec.Unmarshal(buf, &testMap); err != nil { + return fmt.Errorf("failed to unmarshal err: %w", err) + } + default: + return st.Err() + } + } + case interface{ GRPCStatus() *status.Status }: + st := v.GRPCStatus() + me := errors.Parse(st.Message()) + if me.Code != 0 { + if err = expectCodec.Unmarshal([]byte(me.Error()), &testMap); err != nil { + return fmt.Errorf("failed to unmarshal err: %w", err) + } + break + } + case *status.Status: + me := errors.Parse(v.Message()) + if me.Code != 0 { + if err = expectCodec.Unmarshal([]byte(me.Error()), &testMap); err != nil { + return fmt.Errorf("failed to unmarshal err: %w", err) + } + break + } + for _, se := range v.Details() { + switch ne := se.(type) { + case proto.Message: + buf, err := testCodec.Marshal(ne) + if err != nil { + return fmt.Errorf("failed to marshal err: %w", err) + } + if err = testCodec.Unmarshal(buf, &testMap); err != nil { + return fmt.Errorf("failed to unmarshal err: %w", err) + } + default: + return v.Err() + } + } + } + + if !reflect.DeepEqual(expectMap, testMap) { + return fmt.Errorf("test: %s != rsp: %s", expectMap, testMap) + } + + return nil + } +) + +func FromCSVString(columns []*sqlmock.Column, rows *sqlmock.Rows, s string) *sqlmock.Rows { + res := strings.NewReader(strings.TrimSpace(s)) + csvReader := csv.NewReader(res) + + for { + res, err := csvReader.Read() + if err != nil || res == nil { + break + } + + var row []driver.Value + for i, v := range res { + item := CSVColumnParser(strings.TrimSpace(v)) + if null, nullOk := columns[i].IsNullable(); null && nullOk && item == nil { + row = append(row, nil) + } else { + row = append(row, item) + } + + } + rows = rows.AddRow(row...) + } + + return rows +} + +func CSVColumnParser(s string) []byte { + switch { + case strings.ToLower(s) == "null": + return nil + case s == "": + return nil + } + return []byte(s) +} + +func NewResponseFromFile(rspfile string) (*codec.Frame, error) { + rspbuf, err := os.ReadFile(rspfile) + if err != nil { + return nil, err + } + return &codec.Frame{Data: rspbuf}, nil +} + +func getCodec(codecs map[string]codec.Codec, ext string) (codec.Codec, error) { + var c codec.Codec + if cts, ok := ExtToTypes[ext]; ok { + for _, t := range cts { + if c, ok = codecs[t]; ok { + return c, nil + } + } + } + return nil, ErrUnknownContentType +} + +func getContentType(codecs map[string]codec.Codec, ext string) (string, error) { + if cts, ok := ExtToTypes[ext]; ok { + for _, t := range cts { + if _, ok = codecs[t]; ok { + return t, nil + } + } + } + return "", ErrUnknownContentType +} + +func getExt(name string) string { + ext := filepath.Ext(name) + if len(ext) > 0 && ext[0] == '.' { + ext = ext[1:] + } + return ext +} + +func getNameWithoutExt(name string) string { + return strings.TrimSuffix(name, filepath.Ext(name)) +} + +func NewRequestFromFile(c client.Client, reqfile string) (client.Request, error) { + reqbuf, err := os.ReadFile(reqfile) + if err != nil { + return nil, err + } + + endpoint := path.Base(path.Dir(reqfile)) + if idx := strings.Index(endpoint, "_"); idx > 0 { + endpoint = endpoint[idx+1:] + } + ext := getExt(reqfile) + + ct, err := getContentType(c.Options().Codecs, ext) + if err != nil { + return nil, err + } + + req := c.NewRequest("test", endpoint, &codec.Frame{Data: reqbuf}, client.RequestContentType(ct)) + + return req, nil +} + +func SQLFromFile(m sqlmock.Sqlmock, name string) error { + fp, err := os.Open(name) + if err != nil { + return err + } + defer fp.Close() + return SQLFromReader(m, fp) +} + +func SQLFromBytes(m sqlmock.Sqlmock, buf []byte) error { + return SQLFromReader(m, bytes.NewReader(buf)) +} + +func SQLFromString(m sqlmock.Sqlmock, buf string) error { + return SQLFromReader(m, strings.NewReader(buf)) +} + +func SQLFromReader(m sqlmock.Sqlmock, r io.Reader) error { + var rows *sqlmock.Rows + var exp *sqlmock.ExpectedQuery + var columns []*sqlmock.Column + + br := bufio.NewReader(r) + + for { + s, err := br.ReadString('\n') + if err != nil && err != io.EOF { + return err + } else if err == io.EOF && len(s) == 0 { + if rows != nil && exp != nil { + exp.WillReturnRows(rows) + } + return nil + } + + if s[0] != '#' { + r := csv.NewReader(strings.NewReader(s)) + r.Comma = ',' + var records [][]string + records, err = r.ReadAll() + if err != nil { + return err + } + if rows == nil && len(columns) > 0 { + rows = m.NewRowsWithColumnDefinition(columns...) + } else { + for idx := 0; idx < len(records); idx++ { + if len(columns) == 0 { + return fmt.Errorf("csv file not valid, does not have %q line", "# columns ") + } + rows = FromCSVString(columns, rows, strings.Join(records[idx], ",")) + } + } + continue + } + + if rows != nil { + exp.WillReturnRows(rows) + rows = nil + } + + switch { + case strings.HasPrefix(strings.ToLower(s[2:]), "columns"): + for _, field := range strings.Split(s[2+len("columns")+1:], ",") { + args := strings.Split(field, "|") + + column := sqlmock.NewColumn(args[0]).Nullable(false) + + if len(args) > 1 { + for _, arg := range args { + switch arg { + case "BOOLEAN", "BOOL": + column = column.OfType("BOOL", false) + case "NUMBER", "DECIMAL": + column = column.OfType("DECIMAL", float64(0.0)).WithPrecisionAndScale(10, 4) + case "VARCHAR": + column = column.OfType("VARCHAR", nil) + case "NULL": + column = column.Nullable(true) + } + } + } + + columns = append(columns, column) + } + case strings.HasPrefix(strings.ToLower(s[2:]), "begin"): + m.ExpectBegin() + case strings.HasPrefix(strings.ToLower(s[2:]), "commit"): + m.ExpectCommit() + case strings.HasPrefix(strings.ToLower(s[2:]), "rollback"): + m.ExpectRollback() + case strings.HasPrefix(strings.ToLower(s[2:]), "exec "): + m.ExpectExec(s[2+len("exec "):]) + case strings.HasPrefix(strings.ToLower(s[2:]), "query "): + exp = m.ExpectQuery(s[2+len("query "):]) + } + } +} + +func Run(ctx context.Context, c client.Client, m sqlmock.Sqlmock, dir string, exts []string) error { + tcases, err := GetCases(dir, exts) + if err != nil { + return err + } + + g, gctx := errgroup.WithContext(ctx) + if !strings.Contains(dir, "parallel") { + g.SetLimit(1) + } + + for _, tcase := range tcases { + + for _, dbfile := range tcase.dbfiles { + if err = SQLFromFile(m, dbfile); err != nil { + return err + } + } + + tc := tcase + g.Go(func() error { + var xrid string + var gerr error + + treq, err := NewRequestFromFile(c, tc.reqfile) + if err != nil { + gerr = fmt.Errorf("failed to read request from file %s err: %w", tc.reqfile, err) + return gerr + } + + xrid = fmt.Sprintf("%s-%d", treq.Endpoint(), time.Now().Unix()) + + defer func() { + if gerr == nil { + fmt.Printf("test %s xrid: %s status: success\n", filepath.Dir(tc.reqfile), xrid) + } else { + fmt.Printf("test %s xrid: %s status: failure error: %v\n", filepath.Dir(tc.reqfile), xrid, err) + } + }() + + data := &codec.Frame{} + md := metadata.New(1) + md.Set("X-Request-Id", xrid) + cerr := c.Call(metadata.NewOutgoingContext(gctx, md), treq, data, client.WithContentType(treq.ContentType())) + + var rspfile string + + if tc.errfile != "" { + rspfile = tc.errfile + } else if tc.rspfile != "" { + rspfile = tc.rspfile + } else { + gerr = fmt.Errorf("errfile and rspfile is empty") + return gerr + } + + expectRsp, err := NewResponseFromFile(rspfile) + if err != nil { + gerr = fmt.Errorf("failed to read response from file %s err: %w", rspfile, err) + return gerr + } + + testCodec, err := getCodec(Codecs, getExt(tc.reqfile)) + if err != nil { + gerr = fmt.Errorf("failed to get response file codec err: %w", err) + return gerr + } + + expectCodec, err := getCodec(Codecs, getExt(rspfile)) + if err != nil { + gerr = fmt.Errorf("failed to get response file codec err: %w", err) + return gerr + } + + if cerr == nil && tc.errfile != "" { + gerr = fmt.Errorf("expected err %s not happened", expectRsp.Data) + return gerr + } else if cerr != nil && tc.errfile != "" { + if err = ResponseCompareFunc(expectRsp.Data, cerr, expectCodec, testCodec); err != nil { + gerr = err + return gerr + } + } else if cerr != nil && tc.errfile == "" { + gerr = cerr + return gerr + } else if cerr == nil && tc.errfile == "" { + if err = ResponseCompareFunc(expectRsp.Data, data, expectCodec, testCodec); err != nil { + gerr = err + return gerr + } + } + + /* + cf, err := getCodec(c.Options().Codecs, getExt(tc.rspfile)) + if err != nil { + return err + } + */ + + return nil + }) + + } + + return g.Wait() +} + +type Case struct { + dbfiles []string + reqfile string + rspfile string + errfile string +} + +func GetCases(dir string, exts []string) ([]Case, error) { + var tcases []Case + entries, err := os.ReadDir(dir) + if len(entries) == 0 && err != nil { + return tcases, err + } + + if exts == nil { + exts = DefaultExts + } + + var dirs []string + var dbfiles []string + var reqfile, rspfile, errfile string + + for _, entry := range entries { + if entry.IsDir() { + dirs = append(dirs, filepath.Join(dir, entry.Name())) + continue + } + if info, err := entry.Info(); err != nil { + return tcases, err + } else if !info.Mode().IsRegular() { + continue + } + + for _, ext := range exts { + if getExt(entry.Name()) == ext { + name := getNameWithoutExt(entry.Name()) + switch { + case strings.HasSuffix(name, "_db"): + dbfiles = append(dbfiles, filepath.Join(dir, entry.Name())) + case strings.HasSuffix(name, "_req"): + reqfile = filepath.Join(dir, entry.Name()) + case strings.HasSuffix(name, "_rsp"): + rspfile = filepath.Join(dir, entry.Name()) + case strings.HasSuffix(name, "_err"): + errfile = filepath.Join(dir, entry.Name()) + } + } + } + } + + if reqfile != "" && (rspfile != "" || errfile != "") { + tcases = append(tcases, Case{dbfiles: dbfiles, reqfile: reqfile, rspfile: rspfile, errfile: errfile}) + } + + for _, dir = range dirs { + ntcases, err := GetCases(dir, exts) + if len(ntcases) == 0 && err != nil { + return tcases, err + } else if len(ntcases) == 0 { + continue + } + tcases = append(tcases, ntcases...) + } + + return tcases, nil +} diff --git a/util/test/test_test.go b/util/test/test_test.go new file mode 100644 index 00000000..c72bac63 --- /dev/null +++ b/util/test/test_test.go @@ -0,0 +1,72 @@ +package test + +import ( + "context" + "testing" + + "github.com/DATA-DOG/go-sqlmock" +) + +func Test_SQLFromFile(t *testing.T) { + ctx := context.TODO() + db, c, err := sqlmock.New() + if err != nil { + t.Fatal(err) + } + defer db.Close() + + if err = SQLFromFile(c, "testdata/result/01_firstcase/Call_db.csv"); err != nil { + t.Fatal(err) + } + + tx, err := db.BeginTx(ctx, nil) + if err != nil { + t.Fatal(err) + } + + rows, err := tx.QueryContext(ctx, "select * from test;") + if err != nil { + t.Fatal(err) + } + for rows.Next() { + var id int64 + var name string + err = rows.Scan(&id, &name) + if err != nil { + t.Fatal(err) + } + if id != 1 || name != "test" { + t.Fatalf("invalid rows %v %v", id, name) + } + } + + if err = rows.Close(); err != nil { + t.Fatal(err) + } + + if err = rows.Err(); err != nil { + t.Fatal(err) + } + + if err = tx.Commit(); err != nil { + t.Fatal(err) + } + if err = c.ExpectationsWereMet(); err != nil { + t.Fatal(err) + } +} + +func Test_GetCases(t *testing.T) { + files, err := GetCases("testdata/", nil) + if err != nil { + t.Fatal(err) + } + + if len(files) == 0 { + t.Fatalf("no files matching") + } + + if n := len(files); n != 1 { + t.Fatalf("invalid number of test cases %d", n) + } +} diff --git a/util/test/testdata/result/01_firstcase/Call_db.csv b/util/test/testdata/result/01_firstcase/Call_db.csv new file mode 100644 index 00000000..076d4930 --- /dev/null +++ b/util/test/testdata/result/01_firstcase/Call_db.csv @@ -0,0 +1,6 @@ +# begin +# query select \* from test; +# columns id|VARCHAR,name|VARCHAR +id,name +1,test +# commit \ No newline at end of file diff --git a/util/test/testdata/result/01_firstcase/Call_req.json b/util/test/testdata/result/01_firstcase/Call_req.json new file mode 100644 index 00000000..9e26dfee --- /dev/null +++ b/util/test/testdata/result/01_firstcase/Call_req.json @@ -0,0 +1 @@ +{} \ No newline at end of file diff --git a/util/test/testdata/result/01_firstcase/Call_rsp.json b/util/test/testdata/result/01_firstcase/Call_rsp.json new file mode 100644 index 00000000..9e26dfee --- /dev/null +++ b/util/test/testdata/result/01_firstcase/Call_rsp.json @@ -0,0 +1 @@ +{} \ No newline at end of file