util/test/test.go

@@ -0,0 +1,505 @@
+package test
+
+import (
+	"bufio"
+	"bytes"
+	"context"
+	"database/sql/driver"
+	"encoding/csv"
+	"fmt"
+	"io"
+	"os"
+	"path"
+	"path/filepath"
+	"reflect"
+	"strings"
+	"time"
+
+	sqlmock "github.com/DATA-DOG/go-sqlmock"
+	"go.unistack.org/micro/v3/client"
+	"go.unistack.org/micro/v3/codec"
+	"go.unistack.org/micro/v3/errors"
+	"go.unistack.org/micro/v3/metadata"
+	"golang.org/x/sync/errgroup"
+	"google.golang.org/grpc/status"
+	"google.golang.org/protobuf/proto"
+)
+
+var ErrUnknownContentType = fmt.Errorf("unknown content type")
+
+type Extension struct {
+	Ext []string
+}
+
+var (
+	// ExtToTypes map file extension to content type
+	ExtToTypes = map[string][]string{
+		"json":  {"application/json", "application/grpc+json"},
+		"yaml":  {"application/yaml", "application/yml", "text/yaml", "text/yml"},
+		"yml":   {"application/yaml", "application/yml", "text/yaml", "text/yml"},
+		"proto": {"application/grpc", "application/grpc+proto", "application/proto"},
+	}
+	// DefaultExts specifies default file extensions to load data
+	DefaultExts = []string{"csv", "json", "yaml", "yml", "proto"}
+	// Codecs map to detect codec for test file or request content type
+	Codecs map[string]codec.Codec
+
+	// ResponseCompareFunc used to compare actual response with test case data
+	ResponseCompareFunc = func(expectRsp []byte, testRsp interface{}, expectCodec codec.Codec, testCodec codec.Codec) error {
+		var err error
+
+		expectMap := make(map[string]interface{})
+		if err = expectCodec.Unmarshal(expectRsp, &expectMap); err != nil {
+			return fmt.Errorf("failed to unmarshal err: %w", err)
+		}
+
+		testMap := make(map[string]interface{})
+		switch v := testRsp.(type) {
+		case *codec.Frame:
+			if err = testCodec.Unmarshal(v.Data, &testMap); err != nil {
+				return fmt.Errorf("failed to unmarshal err: %w", err)
+			}
+		case *errors.Error:
+			if err = expectCodec.Unmarshal([]byte(v.Error()), &testMap); err != nil {
+				return fmt.Errorf("failed to unmarshal err: %w", err)
+			}
+		case error:
+			st, ok := status.FromError(v)
+			if !ok {
+				return v
+			}
+			me := errors.Parse(st.Message())
+			if me.Code != 0 {
+				if err = expectCodec.Unmarshal([]byte(me.Error()), &testMap); err != nil {
+					return fmt.Errorf("failed to unmarshal err: %w", err)
+				}
+				break
+			}
+			for _, se := range st.Details() {
+				switch ne := se.(type) {
+				case proto.Message:
+					buf, err := testCodec.Marshal(ne)
+					if err != nil {
+						return fmt.Errorf("failed to marshal err: %w", err)
+					}
+					if err = testCodec.Unmarshal(buf, &testMap); err != nil {
+						return fmt.Errorf("failed to unmarshal err: %w", err)
+					}
+				default:
+					return st.Err()
+				}
+			}
+		case interface{ GRPCStatus() *status.Status }:
+			st := v.GRPCStatus()
+			me := errors.Parse(st.Message())
+			if me.Code != 0 {
+				if err = expectCodec.Unmarshal([]byte(me.Error()), &testMap); err != nil {
+					return fmt.Errorf("failed to unmarshal err: %w", err)
+				}
+				break
+			}
+		case *status.Status:
+			me := errors.Parse(v.Message())
+			if me.Code != 0 {
+				if err = expectCodec.Unmarshal([]byte(me.Error()), &testMap); err != nil {
+					return fmt.Errorf("failed to unmarshal err: %w", err)
+				}
+				break
+			}
+			for _, se := range v.Details() {
+				switch ne := se.(type) {
+				case proto.Message:
+					buf, err := testCodec.Marshal(ne)
+					if err != nil {
+						return fmt.Errorf("failed to marshal err: %w", err)
+					}
+					if err = testCodec.Unmarshal(buf, &testMap); err != nil {
+						return fmt.Errorf("failed to unmarshal err: %w", err)
+					}
+				default:
+					return v.Err()
+				}
+			}
+		}
+
+		if !reflect.DeepEqual(expectMap, testMap) {
+			return fmt.Errorf("test: %s != rsp: %s", expectMap, testMap)
+		}
+
+		return nil
+	}
+)
+
+func FromCSVString(columns []*sqlmock.Column, rows *sqlmock.Rows, s string) *sqlmock.Rows {
+	res := strings.NewReader(strings.TrimSpace(s))
+	csvReader := csv.NewReader(res)
+
+	for {
+		res, err := csvReader.Read()
+		if err != nil || res == nil {
+			break
+		}
+
+		var row []driver.Value
+		for i, v := range res {
+			item := CSVColumnParser(strings.TrimSpace(v))
+			if null, nullOk := columns[i].IsNullable(); null && nullOk && item == nil {
+				row = append(row, nil)
+			} else {
+				row = append(row, item)
+			}
+
+		}
+		rows = rows.AddRow(row...)
+	}
+
+	return rows
+}
+
+func CSVColumnParser(s string) []byte {
+	switch {
+	case strings.ToLower(s) == "null":
+		return nil
+	case s == "":
+		return nil
+	}
+	return []byte(s)
+}
+
+func NewResponseFromFile(rspfile string) (*codec.Frame, error) {
+	rspbuf, err := os.ReadFile(rspfile)
+	if err != nil {
+		return nil, err
+	}
+	return &codec.Frame{Data: rspbuf}, nil
+}
+
+func getCodec(codecs map[string]codec.Codec, ext string) (codec.Codec, error) {
+	var c codec.Codec
+	if cts, ok := ExtToTypes[ext]; ok {
+		for _, t := range cts {
+			if c, ok = codecs[t]; ok {
+				return c, nil
+			}
+		}
+	}
+	return nil, ErrUnknownContentType
+}
+
+func getContentType(codecs map[string]codec.Codec, ext string) (string, error) {
+	if cts, ok := ExtToTypes[ext]; ok {
+		for _, t := range cts {
+			if _, ok = codecs[t]; ok {
+				return t, nil
+			}
+		}
+	}
+	return "", ErrUnknownContentType
+}
+
+func getExt(name string) string {
+	ext := filepath.Ext(name)
+	if len(ext) > 0 && ext[0] == '.' {
+		ext = ext[1:]
+	}
+	return ext
+}
+
+func getNameWithoutExt(name string) string {
+	return strings.TrimSuffix(name, filepath.Ext(name))
+}
+
+func NewRequestFromFile(c client.Client, reqfile string) (client.Request, error) {
+	reqbuf, err := os.ReadFile(reqfile)
+	if err != nil {
+		return nil, err
+	}
+
+	endpoint := path.Base(path.Dir(reqfile))
+	if idx := strings.Index(endpoint, "_"); idx > 0 {
+		endpoint = endpoint[idx+1:]
+	}
+	ext := getExt(reqfile)
+
+	ct, err := getContentType(c.Options().Codecs, ext)
+	if err != nil {
+		return nil, err
+	}
+
+	req := c.NewRequest("test", endpoint, &codec.Frame{Data: reqbuf}, client.RequestContentType(ct))
+
+	return req, nil
+}
+
+func SQLFromFile(m sqlmock.Sqlmock, name string) error {
+	fp, err := os.Open(name)
+	if err != nil {
+		return err
+	}
+	defer fp.Close()
+	return SQLFromReader(m, fp)
+}
+
+func SQLFromBytes(m sqlmock.Sqlmock, buf []byte) error {
+	return SQLFromReader(m, bytes.NewReader(buf))
+}
+
+func SQLFromString(m sqlmock.Sqlmock, buf string) error {
+	return SQLFromReader(m, strings.NewReader(buf))
+}
+
+func SQLFromReader(m sqlmock.Sqlmock, r io.Reader) error {
+	var rows *sqlmock.Rows
+	var exp *sqlmock.ExpectedQuery
+	var columns []*sqlmock.Column
+
+	br := bufio.NewReader(r)
+
+	for {
+		s, err := br.ReadString('\n')
+		if err != nil && err != io.EOF {
+			return err
+		} else if err == io.EOF && len(s) == 0 {
+			if rows != nil && exp != nil {
+				exp.WillReturnRows(rows)
+			}
+			return nil
+		}
+
+		if s[0] != '#' {
+			r := csv.NewReader(strings.NewReader(s))
+			r.Comma = ','
+			var records [][]string
+			records, err = r.ReadAll()
+			if err != nil {
+				return err
+			}
+			if rows == nil && len(columns) > 0 {
+				rows = m.NewRowsWithColumnDefinition(columns...)
+			} else {
+				for idx := 0; idx < len(records); idx++ {
+					if len(columns) == 0 {
+						return fmt.Errorf("csv file not valid, does not have %q line", "# columns ")
+					}
+					rows = FromCSVString(columns, rows, strings.Join(records[idx], ","))
+				}
+			}
+			continue
+		}
+
+		if rows != nil {
+			exp.WillReturnRows(rows)
+			rows = nil
+		}
+
+		switch {
+		case strings.HasPrefix(strings.ToLower(s[2:]), "columns"):
+			for _, field := range strings.Split(s[2+len("columns")+1:], ",") {
+				args := strings.Split(field, "|")
+
+				column := sqlmock.NewColumn(args[0]).Nullable(false)
+
+				if len(args) > 1 {
+					for _, arg := range args {
+						switch arg {
+						case "BOOLEAN", "BOOL":
+							column = column.OfType("BOOL", false)
+						case "NUMBER", "DECIMAL":
+							column = column.OfType("DECIMAL", float64(0.0)).WithPrecisionAndScale(10, 4)
+						case "VARCHAR":
+							column = column.OfType("VARCHAR", nil)
+						case "NULL":
+							column = column.Nullable(true)
+						}
+					}
+				}
+
+				columns = append(columns, column)
+			}
+		case strings.HasPrefix(strings.ToLower(s[2:]), "begin"):
+			m.ExpectBegin()
+		case strings.HasPrefix(strings.ToLower(s[2:]), "commit"):
+			m.ExpectCommit()
+		case strings.HasPrefix(strings.ToLower(s[2:]), "rollback"):
+			m.ExpectRollback()
+		case strings.HasPrefix(strings.ToLower(s[2:]), "exec "):
+			m.ExpectExec(s[2+len("exec "):])
+		case strings.HasPrefix(strings.ToLower(s[2:]), "query "):
+			exp = m.ExpectQuery(s[2+len("query "):])
+		}
+	}
+}
+
+func Run(ctx context.Context, c client.Client, m sqlmock.Sqlmock, dir string, exts []string) error {
+	tcases, err := GetCases(dir, exts)
+	if err != nil {
+		return err
+	}
+
+	g, gctx := errgroup.WithContext(ctx)
+	if !strings.Contains(dir, "parallel") {
+		g.SetLimit(1)
+	}
+
+	for _, tcase := range tcases {
+
+		for _, dbfile := range tcase.dbfiles {
+			if err = SQLFromFile(m, dbfile); err != nil {
+				return err
+			}
+		}
+
+		tc := tcase
+		g.Go(func() error {
+			var xrid string
+			var gerr error
+
+			treq, err := NewRequestFromFile(c, tc.reqfile)
+			if err != nil {
+				gerr = fmt.Errorf("failed to read request from file %s err: %w", tc.reqfile, err)
+				return gerr
+			}
+
+			xrid = fmt.Sprintf("%s-%d", treq.Endpoint(), time.Now().Unix())
+
+			defer func() {
+				if gerr == nil {
+					fmt.Printf("test %s xrid: %s status: success\n", filepath.Dir(tc.reqfile), xrid)
+				} else {
+					fmt.Printf("test %s xrid: %s status: failure error: %v\n", filepath.Dir(tc.reqfile), xrid, err)
+				}
+			}()
+
+			data := &codec.Frame{}
+			md := metadata.New(1)
+			md.Set("X-Request-Id", xrid)
+			cerr := c.Call(metadata.NewOutgoingContext(gctx, md), treq, data, client.WithContentType(treq.ContentType()))
+
+			var rspfile string
+
+			if tc.errfile != "" {
+				rspfile = tc.errfile
+			} else if tc.rspfile != "" {
+				rspfile = tc.rspfile
+			} else {
+				gerr = fmt.Errorf("errfile and rspfile is empty")
+				return gerr
+			}
+
+			expectRsp, err := NewResponseFromFile(rspfile)
+			if err != nil {
+				gerr = fmt.Errorf("failed to read response from file %s err: %w", rspfile, err)
+				return gerr
+			}
+
+			testCodec, err := getCodec(Codecs, getExt(tc.reqfile))
+			if err != nil {
+				gerr = fmt.Errorf("failed to get response file codec err: %w", err)
+				return gerr
+			}
+
+			expectCodec, err := getCodec(Codecs, getExt(rspfile))
+			if err != nil {
+				gerr = fmt.Errorf("failed to get response file codec err: %w", err)
+				return gerr
+			}
+
+			if cerr == nil && tc.errfile != "" {
+				gerr = fmt.Errorf("expected err %s not happened", expectRsp.Data)
+				return gerr
+			} else if cerr != nil && tc.errfile != "" {
+				if err = ResponseCompareFunc(expectRsp.Data, cerr, expectCodec, testCodec); err != nil {
+					gerr = err
+					return gerr
+				}
+			} else if cerr != nil && tc.errfile == "" {
+				gerr = cerr
+				return gerr
+			} else if cerr == nil && tc.errfile == "" {
+				if err = ResponseCompareFunc(expectRsp.Data, data, expectCodec, testCodec); err != nil {
+					gerr = err
+					return gerr
+				}
+			}
+
+			/*
+				cf, err := getCodec(c.Options().Codecs, getExt(tc.rspfile))
+				if err != nil {
+					return err
+				}
+			*/
+
+			return nil
+		})
+
+	}
+
+	return g.Wait()
+}
+
+type Case struct {
+	dbfiles []string
+	reqfile string
+	rspfile string
+	errfile string
+}
+
+func GetCases(dir string, exts []string) ([]Case, error) {
+	var tcases []Case
+	entries, err := os.ReadDir(dir)
+	if len(entries) == 0 && err != nil {
+		return tcases, err
+	}
+
+	if exts == nil {
+		exts = DefaultExts
+	}
+
+	var dirs []string
+	var dbfiles []string
+	var reqfile, rspfile, errfile string
+
+	for _, entry := range entries {
+		if entry.IsDir() {
+			dirs = append(dirs, filepath.Join(dir, entry.Name()))
+			continue
+		}
+		if info, err := entry.Info(); err != nil {
+			return tcases, err
+		} else if !info.Mode().IsRegular() {
+			continue
+		}
+
+		for _, ext := range exts {
+			if getExt(entry.Name()) == ext {
+				name := getNameWithoutExt(entry.Name())
+				switch {
+				case strings.HasSuffix(name, "_db"):
+					dbfiles = append(dbfiles, filepath.Join(dir, entry.Name()))
+				case strings.HasSuffix(name, "_req"):
+					reqfile = filepath.Join(dir, entry.Name())
+				case strings.HasSuffix(name, "_rsp"):
+					rspfile = filepath.Join(dir, entry.Name())
+				case strings.HasSuffix(name, "_err"):
+					errfile = filepath.Join(dir, entry.Name())
+				}
+			}
+		}
+	}
+
+	if reqfile != "" && (rspfile != "" || errfile != "") {
+		tcases = append(tcases, Case{dbfiles: dbfiles, reqfile: reqfile, rspfile: rspfile, errfile: errfile})
+	}
+
+	for _, dir = range dirs {
+		ntcases, err := GetCases(dir, exts)
+		if len(ntcases) == 0 && err != nil {
+			return tcases, err
+		} else if len(ntcases) == 0 {
+			continue
+		}
+		tcases = append(tcases, ntcases...)
+	}
+
+	return tcases, nil
+}

util/test/test_test.go

@@ -0,0 +1,72 @@
+package test
+
+import (
+	"context"
+	"testing"
+
+	"github.com/DATA-DOG/go-sqlmock"
+)
+
+func Test_SQLFromFile(t *testing.T) {
+	ctx := context.TODO()
+	db, c, err := sqlmock.New()
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer db.Close()
+
+	if err = SQLFromFile(c, "testdata/result/01_firstcase/Call_db.csv"); err != nil {
+		t.Fatal(err)
+	}
+
+	tx, err := db.BeginTx(ctx, nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	rows, err := tx.QueryContext(ctx, "select * from test;")
+	if err != nil {
+		t.Fatal(err)
+	}
+	for rows.Next() {
+		var id int64
+		var name string
+		err = rows.Scan(&id, &name)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if id != 1 || name != "test" {
+			t.Fatalf("invalid rows %v %v", id, name)
+		}
+	}
+
+	if err = rows.Close(); err != nil {
+		t.Fatal(err)
+	}
+
+	if err = rows.Err(); err != nil {
+		t.Fatal(err)
+	}
+
+	if err = tx.Commit(); err != nil {
+		t.Fatal(err)
+	}
+	if err = c.ExpectationsWereMet(); err != nil {
+		t.Fatal(err)
+	}
+}
+
+func Test_GetCases(t *testing.T) {
+	files, err := GetCases("testdata/", nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if len(files) == 0 {
+		t.Fatalf("no files matching")
+	}
+
+	if n := len(files); n != 1 {
+		t.Fatalf("invalid number of test cases %d", n)
+	}
+}

util/test/testdata/result/01_firstcase/Call_db.csv

@@ -0,0 +1,6 @@
+# begin
+# query select \* from test;
+# columns id|VARCHAR,name|VARCHAR
+id,name
+1,test
+# commit

util/test/testdata/result/01_firstcase/Call_req.json

@@ -0,0 +1 @@
+{}

util/test/testdata/result/01_firstcase/Call_rsp.json

@@ -0,0 +1 @@
+{}