From 19e3288b1d4f67e4a59f36482439b4c7e8dd8cad Mon Sep 17 00:00:00 2001 From: Vasiliy Tolstov Date: Fri, 19 Sep 2025 11:39:03 +0300 Subject: [PATCH 1/6] initial cluster sql support Signed-off-by: Vasiliy Tolstov --- cluster/sql/cluster.go | 531 ++++++++++++++++++++++++++++++++++++ cluster/sql/cluster_test.go | 171 ++++++++++++ 2 files changed, 702 insertions(+) create mode 100644 cluster/sql/cluster.go create mode 100644 cluster/sql/cluster_test.go diff --git a/cluster/sql/cluster.go b/cluster/sql/cluster.go new file mode 100644 index 00000000..e731e2c1 --- /dev/null +++ b/cluster/sql/cluster.go @@ -0,0 +1,531 @@ +package sql + +import ( + "context" + "database/sql" + "errors" + "fmt" + "math" + "reflect" + "time" + "unsafe" + + "golang.yandex/hasql/v2" +) + +var errNoAliveNodes = errors.New("no alive nodes") + +func newSQLRowError() *sql.Row { + row := &sql.Row{} + t := reflect.TypeOf(row).Elem() + field, _ := t.FieldByName("err") + rowPtr := unsafe.Pointer(row) + errFieldPtr := unsafe.Pointer(uintptr(rowPtr) + field.Offset) + errPtr := (*error)(errFieldPtr) + *errPtr = errNoAliveNodes + return row +} + +type ClusterQuerier interface { + Querier + WaitForNodes(ctx context.Context, criterion ...hasql.NodeStateCriterion) error +} + +type Querier interface { + // Basic connection methods + PingContext(ctx context.Context) error + Close() error + + // Query methods with context + ExecContext(ctx context.Context, query string, args ...interface{}) (sql.Result, error) + QueryContext(ctx context.Context, query string, args ...interface{}) (*sql.Rows, error) + QueryRowContext(ctx context.Context, query string, args ...interface{}) *sql.Row + + // Prepared statements with context + PrepareContext(ctx context.Context, query string) (*sql.Stmt, error) + + // Transaction management with context + BeginTx(ctx context.Context, opts *sql.TxOptions) (*sql.Tx, error) + + // Connection pool management + SetConnMaxLifetime(d time.Duration) + SetConnMaxIdleTime(d time.Duration) + SetMaxOpenConns(n int) + SetMaxIdleConns(n int) + Stats() sql.DBStats + + Conn(ctx context.Context) (*sql.Conn, error) +} + +var ( + ErrClusterChecker = errors.New("cluster node checker required") + ErrClusterDiscoverer = errors.New("cluster node discoverer required") + ErrClusterPicker = errors.New("cluster node picker required") +) + +type Cluster struct { + hasql *hasql.Cluster[Querier] + options ClusterOptions +} + +// NewCluster returns Querier that provides cluster of nodes +func NewCluster[T Querier](opts ...ClusterOption) (ClusterQuerier, error) { + options := ClusterOptions{Context: context.Background()} + for _, opt := range opts { + opt(&options) + } + if options.NodeChecker == nil { + return nil, ErrClusterChecker + } + if options.NodeDiscoverer == nil { + return nil, ErrClusterDiscoverer + } + if options.NodePicker == nil { + return nil, ErrClusterPicker + } + + if options.Retries < 1 { + options.Retries = 1 + } + + if options.NodeStateCriterion == 0 { + options.NodeStateCriterion = hasql.Primary + } + + options.Options = append(options.Options, hasql.WithNodePicker(options.NodePicker)) + if p, ok := options.NodePicker.(*CustomPicker[Querier]); ok { + p.opts.Priority = options.NodePriority + } + + c, err := hasql.NewCluster( + options.NodeDiscoverer, + options.NodeChecker, + options.Options..., + ) + if err != nil { + return nil, err + } + + return &Cluster{hasql: c, options: options}, nil +} + +// compile time guard +var _ hasql.NodePicker[Querier] = (*CustomPicker[Querier])(nil) + +type nodeStateCriterionKey struct{} + +// NodeStateCriterion inject hasql.NodeStateCriterion to context +func NodeStateCriterion(ctx context.Context, c hasql.NodeStateCriterion) context.Context { + return context.WithValue(ctx, nodeStateCriterionKey{}, c) +} + +// CustomPickerOptions holds options to pick nodes +type CustomPickerOptions struct { + MaxLag int + Priority map[string]int32 + Retries int +} + +// CustomPickerOption func apply option to CustomPickerOptions +type CustomPickerOption func(*CustomPickerOptions) + +// CustomPickerMaxLag specifies max lag for which node can be used +func CustomPickerMaxLag(n int) CustomPickerOption { + return func(o *CustomPickerOptions) { + o.MaxLag = n + } +} + +// NewCustomPicker creates new node picker +func NewCustomPicker[T Querier](opts ...CustomPickerOption) *CustomPicker[Querier] { + options := CustomPickerOptions{} + for _, o := range opts { + o(&options) + } + return &CustomPicker[Querier]{opts: options} +} + +// CustomPicker holds node picker options +type CustomPicker[T Querier] struct { + opts CustomPickerOptions +} + +// PickNode used to return specific node +func (p *CustomPicker[T]) PickNode(cnodes []hasql.CheckedNode[T]) hasql.CheckedNode[T] { + for _, n := range cnodes { + fmt.Printf("node %s\n", n.Node.String()) + } + return cnodes[0] +} + +func (p *CustomPicker[T]) getPriority(nodeName string) int32 { + if prio, ok := p.opts.Priority[nodeName]; ok { + return prio + } + return math.MaxInt32 // Default to lowest priority +} + +// CompareNodes used to sort nodes +func (p *CustomPicker[T]) CompareNodes(a, b hasql.CheckedNode[T]) int { + fmt.Printf("CompareNodes %s %s\n", a.Node.String(), b.Node.String()) + // Get replication lag values + aLag := a.Info.(interface{ ReplicationLag() int }).ReplicationLag() + bLag := b.Info.(interface{ ReplicationLag() int }).ReplicationLag() + + // First check that lag lower then MaxLag + if aLag > p.opts.MaxLag && bLag > p.opts.MaxLag { + fmt.Printf("CompareNodes aLag > p.opts.MaxLag && bLag > p.opts.MaxLag\n") + return 0 // both are equal + } + + // If one node exceeds MaxLag and the other doesn't, prefer the one that doesn't + if aLag > p.opts.MaxLag { + fmt.Printf("CompareNodes aLag > p.opts.MaxLag\n") + return 1 // b is better + } + if bLag > p.opts.MaxLag { + fmt.Printf("CompareNodes bLag > p.opts.MaxLag\n") + return -1 // a is better + } + + // Get node priorities + aPrio := p.getPriority(a.Node.String()) + bPrio := p.getPriority(b.Node.String()) + + // if both priority equals + if aPrio == bPrio { + fmt.Printf("CompareNodes aPrio == bPrio\n") + // First compare by replication lag + if aLag < bLag { + fmt.Printf("CompareNodes aLag < bLag\n") + return -1 + } + if aLag > bLag { + fmt.Printf("CompareNodes aLag > bLag\n") + return 1 + } + // If replication lag is equal, compare by latency + aLatency := a.Info.(interface{ Latency() time.Duration }).Latency() + bLatency := b.Info.(interface{ Latency() time.Duration }).Latency() + + if aLatency < bLatency { + return -1 + } + if aLatency > bLatency { + return 1 + } + + // If lag and latency is equal + return 0 + } + + // If priorities are different, prefer the node with lower priority value + if aPrio < bPrio { + return -1 + } + + return 1 +} + +// ClusterOptions contains cluster specific options +type ClusterOptions struct { + NodeChecker hasql.NodeChecker + NodePicker hasql.NodePicker[Querier] + NodeDiscoverer hasql.NodeDiscoverer[Querier] + Options []hasql.ClusterOpt[Querier] + Context context.Context + Retries int + NodePriority map[string]int32 + NodeStateCriterion hasql.NodeStateCriterion +} + +// ClusterOption apply cluster options to ClusterOptions +type ClusterOption func(*ClusterOptions) + +// WithClusterNodeChecker pass hasql.NodeChecker to cluster options +func WithClusterNodeChecker(c hasql.NodeChecker) ClusterOption { + return func(o *ClusterOptions) { + o.NodeChecker = c + } +} + +// WithClusterNodePicker pass hasql.NodePicker to cluster options +func WithClusterNodePicker(p hasql.NodePicker[Querier]) ClusterOption { + return func(o *ClusterOptions) { + o.NodePicker = p + } +} + +// WithClusterNodeDiscoverer pass hasql.NodeDiscoverer to cluster options +func WithClusterNodeDiscoverer(d hasql.NodeDiscoverer[Querier]) ClusterOption { + return func(o *ClusterOptions) { + o.NodeDiscoverer = d + } +} + +// WithRetries retry count on other nodes in case of error +func WithRetries(n int) ClusterOption { + return func(o *ClusterOptions) { + o.Retries = n + } +} + +// WithClusterContext pass context.Context to cluster options and used for checks +func WithClusterContext(ctx context.Context) ClusterOption { + return func(o *ClusterOptions) { + o.Context = ctx + } +} + +// WithClusterOptions pass hasql.ClusterOpt +func WithClusterOptions(opts ...hasql.ClusterOpt[Querier]) ClusterOption { + return func(o *ClusterOptions) { + o.Options = append(o.Options, opts...) + } +} + +// WithClusterNodeStateCriterion pass default hasql.NodeStateCriterion +func WithClusterNodeStateCriterion(c hasql.NodeStateCriterion) ClusterOption { + return func(o *ClusterOptions) { + o.NodeStateCriterion = c + } +} + +type ClusterNode struct { + Name string + DB Querier + Priority int32 +} + +// WithClusterNodes create cluster with static NodeDiscoverer +func WithClusterNodes(cns ...ClusterNode) ClusterOption { + return func(o *ClusterOptions) { + nodes := make([]*hasql.Node[Querier], 0, len(cns)) + if o.NodePriority == nil { + o.NodePriority = make(map[string]int32, len(cns)) + } + for _, cn := range cns { + nodes = append(nodes, hasql.NewNode(cn.Name, cn.DB)) + if cn.Priority == 0 { + cn.Priority = math.MaxInt32 + } + o.NodePriority[cn.Name] = cn.Priority + } + o.NodeDiscoverer = hasql.NewStaticNodeDiscoverer(nodes...) + } +} + +func (c *Cluster) BeginTx(ctx context.Context, opts *sql.TxOptions) (*sql.Tx, error) { + var tx *sql.Tx + var err error + + retries := 0 + c.hasql.NodesIter(c.getNodeStateCriterion(ctx))(func(n *hasql.Node[Querier]) bool { + for ; retries < c.options.Retries; retries++ { + if tx, err = n.DB().BeginTx(ctx, opts); err != nil && retries >= c.options.Retries { + return true + } + } + return false + }) + + if tx == nil && err == nil { + err = errNoAliveNodes + } + + return tx, err +} + +func (c *Cluster) Close() error { + return c.hasql.Close() +} + +func (c *Cluster) Conn(ctx context.Context) (*sql.Conn, error) { + var conn *sql.Conn + var err error + + retries := 0 + c.hasql.NodesIter(c.getNodeStateCriterion(ctx))(func(n *hasql.Node[Querier]) bool { + for ; retries < c.options.Retries; retries++ { + if conn, err = n.DB().Conn(ctx); err != nil && retries >= c.options.Retries { + return true + } + } + return false + }) + + if conn == nil && err == nil { + err = errNoAliveNodes + } + + return conn, err +} + +func (c *Cluster) ExecContext(ctx context.Context, query string, args ...interface{}) (sql.Result, error) { + var res sql.Result + var err error + + retries := 0 + c.hasql.NodesIter(c.getNodeStateCriterion(ctx))(func(n *hasql.Node[Querier]) bool { + for ; retries < c.options.Retries; retries++ { + if res, err = n.DB().ExecContext(ctx, query, args...); err != nil && retries >= c.options.Retries { + return true + } + } + return false + }) + + if res == nil && err == nil { + err = errNoAliveNodes + } + + return res, err +} + +func (c *Cluster) PrepareContext(ctx context.Context, query string) (*sql.Stmt, error) { + var res *sql.Stmt + var err error + + retries := 0 + c.hasql.NodesIter(c.getNodeStateCriterion(ctx))(func(n *hasql.Node[Querier]) bool { + for ; retries < c.options.Retries; retries++ { + if res, err = n.DB().PrepareContext(ctx, query); err != nil && retries >= c.options.Retries { + return true + } + } + return false + }) + + if res == nil && err == nil { + err = errNoAliveNodes + } + + return res, err +} + +func (c *Cluster) QueryContext(ctx context.Context, query string, args ...interface{}) (*sql.Rows, error) { + var res *sql.Rows + var err error + + retries := 0 + c.hasql.NodesIter(c.getNodeStateCriterion(ctx))(func(n *hasql.Node[Querier]) bool { + for ; retries < c.options.Retries; retries++ { + if res, err = n.DB().QueryContext(ctx, query); err != nil && err != sql.ErrNoRows && retries >= c.options.Retries { + return true + } + } + return false + }) + + if res == nil && err == nil { + err = errNoAliveNodes + } + + return res, err +} + +func (c *Cluster) QueryRowContext(ctx context.Context, query string, args ...interface{}) *sql.Row { + var res *sql.Row + + retries := 0 + c.hasql.NodesIter(c.getNodeStateCriterion(ctx))(func(n *hasql.Node[Querier]) bool { + for ; retries < c.options.Retries; retries++ { + res = n.DB().QueryRowContext(ctx, query, args...) + if res.Err() == nil { + return false + } else if res.Err() != nil && retries >= c.options.Retries { + return false + } + } + return true + }) + + if res == nil { + res = newSQLRowError() + } + + return res +} + +func (c *Cluster) PingContext(ctx context.Context) error { + var err error + var ok bool + + retries := 0 + c.hasql.NodesIter(c.getNodeStateCriterion(ctx))(func(n *hasql.Node[Querier]) bool { + ok = true + for ; retries < c.options.Retries; retries++ { + if err = n.DB().PingContext(ctx); err != nil && retries >= c.options.Retries { + return true + } + } + return false + }) + + if !ok { + err = errNoAliveNodes + } + + return err +} + +func (c *Cluster) WaitForNodes(ctx context.Context, criterions ...hasql.NodeStateCriterion) error { + for _, criterion := range criterions { + if _, err := c.hasql.WaitForNode(ctx, criterion); err != nil { + return err + } + } + return nil +} + +func (c *Cluster) SetConnMaxLifetime(td time.Duration) { + c.hasql.NodesIter(hasql.NodeStateCriterion(hasql.Alive))(func(n *hasql.Node[Querier]) bool { + n.DB().SetConnMaxIdleTime(td) + return false + }) +} + +func (c *Cluster) SetConnMaxIdleTime(td time.Duration) { + c.hasql.NodesIter(hasql.NodeStateCriterion(hasql.Alive))(func(n *hasql.Node[Querier]) bool { + n.DB().SetConnMaxIdleTime(td) + return false + }) +} + +func (c *Cluster) SetMaxOpenConns(nc int) { + c.hasql.NodesIter(hasql.NodeStateCriterion(hasql.Alive))(func(n *hasql.Node[Querier]) bool { + n.DB().SetMaxOpenConns(nc) + return false + }) +} + +func (c *Cluster) SetMaxIdleConns(nc int) { + c.hasql.NodesIter(hasql.NodeStateCriterion(hasql.Alive))(func(n *hasql.Node[Querier]) bool { + n.DB().SetMaxIdleConns(nc) + return false + }) +} + +func (c *Cluster) Stats() sql.DBStats { + s := sql.DBStats{} + c.hasql.NodesIter(hasql.NodeStateCriterion(hasql.Alive))(func(n *hasql.Node[Querier]) bool { + st := n.DB().Stats() + s.Idle += st.Idle + s.InUse += st.InUse + s.MaxIdleClosed += st.MaxIdleClosed + s.MaxIdleTimeClosed += st.MaxIdleTimeClosed + s.MaxOpenConnections += st.MaxOpenConnections + s.OpenConnections += st.OpenConnections + s.WaitCount += st.WaitCount + s.WaitDuration += st.WaitDuration + return false + }) + return s +} + +func (c *Cluster) getNodeStateCriterion(ctx context.Context) hasql.NodeStateCriterion { + if v, ok := ctx.Value(nodeStateCriterionKey{}).(hasql.NodeStateCriterion); ok { + return v + } + return c.options.NodeStateCriterion +} diff --git a/cluster/sql/cluster_test.go b/cluster/sql/cluster_test.go new file mode 100644 index 00000000..6a1455ba --- /dev/null +++ b/cluster/sql/cluster_test.go @@ -0,0 +1,171 @@ +package sql + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/DATA-DOG/go-sqlmock" + "golang.yandex/hasql/v2" +) + +func TestNewCluster(t *testing.T) { + dbMaster, dbMasterMock, err := sqlmock.New(sqlmock.MonitorPingsOption(true)) + if err != nil { + t.Fatal(err) + } + defer dbMaster.Close() + dbMasterMock.MatchExpectationsInOrder(false) + + dbMasterMock.ExpectQuery(`.*pg_is_in_recovery.*`).WillReturnRows( + sqlmock.NewRowsWithColumnDefinition( + sqlmock.NewColumn("role").OfType("int8", 0), + sqlmock.NewColumn("replication_lag").OfType("int8", 0)). + AddRow(1, 0)). + RowsWillBeClosed(). + WithoutArgs() + + dbMasterMock.ExpectQuery(`SELECT node_name as name`).WillReturnRows( + sqlmock.NewRows([]string{"name"}). + AddRow("master-dc1")) + + dbDRMaster, dbDRMasterMock, err := sqlmock.New(sqlmock.MonitorPingsOption(true)) + if err != nil { + t.Fatal(err) + } + defer dbDRMaster.Close() + dbDRMasterMock.MatchExpectationsInOrder(false) + + dbDRMasterMock.ExpectQuery(`.*pg_is_in_recovery.*`).WillReturnRows( + sqlmock.NewRowsWithColumnDefinition( + sqlmock.NewColumn("role").OfType("int8", 0), + sqlmock.NewColumn("replication_lag").OfType("int8", 0)). + AddRow(2, 40)). + RowsWillBeClosed(). + WithoutArgs() + + dbDRMasterMock.ExpectQuery(`SELECT node_name as name`).WillReturnRows( + sqlmock.NewRows([]string{"name"}). + AddRow("drmaster1-dc2")) + + dbDRMasterMock.ExpectQuery(`SELECT node_name as name`).WillReturnRows( + sqlmock.NewRows([]string{"name"}). + AddRow("drmaster")) + + dbSlaveDC1, dbSlaveDC1Mock, err := sqlmock.New(sqlmock.MonitorPingsOption(true)) + if err != nil { + t.Fatal(err) + } + defer dbSlaveDC1.Close() + dbSlaveDC1Mock.MatchExpectationsInOrder(false) + + dbSlaveDC1Mock.ExpectQuery(`.*pg_is_in_recovery.*`).WillReturnRows( + sqlmock.NewRowsWithColumnDefinition( + sqlmock.NewColumn("role").OfType("int8", 0), + sqlmock.NewColumn("replication_lag").OfType("int8", 0)). + AddRow(2, 50)). + RowsWillBeClosed(). + WithoutArgs() + + dbSlaveDC1Mock.ExpectQuery(`SELECT node_name as name`).WillReturnRows( + sqlmock.NewRows([]string{"name"}). + AddRow("slave-dc1")) + + dbSlaveDC2, dbSlaveDC2Mock, err := sqlmock.New(sqlmock.MonitorPingsOption(true)) + if err != nil { + t.Fatal(err) + } + defer dbSlaveDC2.Close() + dbSlaveDC1Mock.MatchExpectationsInOrder(false) + + dbSlaveDC2Mock.ExpectQuery(`.*pg_is_in_recovery.*`).WillReturnRows( + sqlmock.NewRowsWithColumnDefinition( + sqlmock.NewColumn("role").OfType("int8", 0), + sqlmock.NewColumn("replication_lag").OfType("int8", 0)). + AddRow(2, 50)). + RowsWillBeClosed(). + WithoutArgs() + + dbSlaveDC2Mock.ExpectQuery(`SELECT node_name as name`).WillReturnRows( + sqlmock.NewRows([]string{"name"}). + AddRow("slave-dc1")) + + tctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) + defer cancel() + + c, err := NewCluster[Querier]( + WithClusterContext(tctx), + WithClusterNodeChecker(hasql.PostgreSQLChecker), + WithClusterNodePicker(NewCustomPicker[Querier]( + CustomPickerMaxLag(100), + )), + WithClusterNodes( + ClusterNode{"slave-dc1", dbSlaveDC1, 1}, + ClusterNode{"master-dc1", dbMaster, 1}, + ClusterNode{"slave-dc2", dbSlaveDC2, 2}, + ClusterNode{"drmaster1-dc2", dbDRMaster, 0}, + ), + WithClusterOptions( + hasql.WithUpdateInterval[Querier](2*time.Second), + hasql.WithUpdateTimeout[Querier](1*time.Second), + ), + ) + if err != nil { + t.Fatal(err) + } + defer c.Close() + + if err = c.WaitForNodes(tctx, hasql.Primary, hasql.Standby); err != nil { + t.Fatal(err) + } + + time.Sleep(500 * time.Millisecond) + + node1Name := "" + fmt.Printf("check for Standby\n") + if row := c.QueryRowContext(NodeStateCriterion(tctx, hasql.Standby), "SELECT node_name as name"); row.Err() != nil { + t.Fatal(row.Err()) + } else if err = row.Scan(&node1Name); err != nil { + t.Fatal(err) + } else if "slave-dc1" != node1Name { + t.Fatalf("invalid node name %s != %s", "slave-dc1", node1Name) + } + + dbSlaveDC1Mock.ExpectQuery(`SELECT node_name as name`).WillReturnRows( + sqlmock.NewRows([]string{"name"}). + AddRow("slave-dc1")) + + node2Name := "" + fmt.Printf("check for PreferStandby\n") + if row := c.QueryRowContext(NodeStateCriterion(tctx, hasql.PreferStandby), "SELECT node_name as name"); row.Err() != nil { + t.Fatal(row.Err()) + } else if err = row.Scan(&node2Name); err != nil { + t.Fatal(err) + } else if "slave-dc1" != node2Name { + t.Fatalf("invalid node name %s != %s", "slave-dc1", node2Name) + } + + node3Name := "" + fmt.Printf("check for PreferPrimary\n") + if row := c.QueryRowContext(NodeStateCriterion(tctx, hasql.PreferPrimary), "SELECT node_name as name"); row.Err() != nil { + t.Fatal(row.Err()) + } else if err = row.Scan(&node3Name); err != nil { + t.Fatal(err) + } else if "master-dc1" != node3Name { + t.Fatalf("invalid node name %s != %s", "master-dc1", node3Name) + } + + dbSlaveDC1Mock.ExpectQuery(`.*`).WillReturnRows(sqlmock.NewRows([]string{"role"}).RowError(1, fmt.Errorf("row error"))) + + time.Sleep(2 * time.Second) + + fmt.Printf("check for PreferStandby\n") + if row := c.QueryRowContext(NodeStateCriterion(tctx, hasql.PreferStandby), "SELECT node_name as name"); row.Err() == nil { + t.Fatal("must return error") + } + + if dbMasterErr := dbMasterMock.ExpectationsWereMet(); dbMasterErr != nil { + t.Error(dbMasterErr) + } +} -- 2.49.1 From 4de9431df0b3094bbc5060714a72ab598f716ec3 Mon Sep 17 00:00:00 2001 From: Vasiliy Tolstov Date: Fri, 19 Sep 2025 11:39:26 +0300 Subject: [PATCH 2/6] initial cluster sql support Signed-off-by: Vasiliy Tolstov --- go.mod | 3 ++- go.sum | 2 ++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 24edf09b..36be164c 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module go.unistack.org/micro/v3 -go 1.22.0 +go 1.23.0 require ( dario.cat/mergo v1.0.1 @@ -15,6 +15,7 @@ require ( go.uber.org/automaxprocs v1.6.0 go.unistack.org/micro-proto/v3 v3.4.1 golang.org/x/sync v0.10.0 + golang.yandex/hasql/v2 v2.1.0 google.golang.org/grpc v1.69.2 google.golang.org/protobuf v1.36.1 gopkg.in/yaml.v3 v3.0.1 diff --git a/go.sum b/go.sum index 72be286c..70d1ac2f 100644 --- a/go.sum +++ b/go.sum @@ -89,6 +89,8 @@ golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= +golang.yandex/hasql/v2 v2.1.0 h1:7CaFFWeHoK5TvA+QvZzlKHlIN5sqNpqM8NSrXskZD/k= +golang.yandex/hasql/v2 v2.1.0/go.mod h1:3Au1AxuJDCTXmS117BpbI6e+70kGWeyLR1qJAH6HdtA= google.golang.org/genproto/googleapis/rpc v0.0.0-20241216192217-9240e9c98484 h1:Z7FRVJPSMaHQxD0uXU8WdgFh8PseLM8Q8NzhnpMrBhQ= google.golang.org/genproto/googleapis/rpc v0.0.0-20241216192217-9240e9c98484/go.mod h1:lcTa1sDdWEIHMWlITnIczmw5w60CF9ffkb8Z+DVmmjA= google.golang.org/grpc v1.69.2 h1:U3S9QEtbXC0bYNvRtcoklF3xGtLViumSYxWykJS+7AU= -- 2.49.1 From 3cef42f77620e7533e0d66c20ac786766f3a7a2e Mon Sep 17 00:00:00 2001 From: vtolstov Date: Fri, 19 Sep 2025 08:44:33 +0000 Subject: [PATCH 3/6] Apply Code Coverage Badge --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index cca89a9b..754587ed 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ # Micro -![Coverage](https://img.shields.io/badge/Coverage-33.7%25-yellow) +![Coverage](https://img.shields.io/badge/Coverage-34.0%25-yellow) [![License](https://img.shields.io/:license-apache-blue.svg)](https://opensource.org/licenses/Apache-2.0) [![Doc](https://img.shields.io/badge/go.dev-reference-007d9c?logo=go&logoColor=white&style=flat-square)](https://pkg.go.dev/go.unistack.org/micro/v3?tab=overview) [![Status](https://git.unistack.org/unistack-org/micro/actions/workflows/job_tests.yml/badge.svg?branch=v3)](https://git.unistack.org/unistack-org/micro/actions?query=workflow%3Abuild+branch%3Av3+event%3Apush) -- 2.49.1 From ffd5e416c8402709812e0493bcadf9b0f730ef99 Mon Sep 17 00:00:00 2001 From: Vasiliy Tolstov Date: Sat, 20 Sep 2025 22:58:33 +0300 Subject: [PATCH 4/6] split files Signed-off-by: Vasiliy Tolstov --- cluster/hasql/cluster.go | 235 +++++++++++++++++++++++++++ cluster/hasql/cluster_test.go | 171 ++++++++++++++++++++ cluster/hasql/db.go | 25 +++ cluster/hasql/driver.go | 295 ++++++++++++++++++++++++++++++++++ cluster/hasql/driver_test.go | 141 ++++++++++++++++ cluster/hasql/error.go | 10 ++ cluster/hasql/options.go | 110 +++++++++++++ cluster/hasql/picker.go | 113 +++++++++++++ 8 files changed, 1100 insertions(+) create mode 100644 cluster/hasql/cluster.go create mode 100644 cluster/hasql/cluster_test.go create mode 100644 cluster/hasql/db.go create mode 100644 cluster/hasql/driver.go create mode 100644 cluster/hasql/driver_test.go create mode 100644 cluster/hasql/error.go create mode 100644 cluster/hasql/options.go create mode 100644 cluster/hasql/picker.go diff --git a/cluster/hasql/cluster.go b/cluster/hasql/cluster.go new file mode 100644 index 00000000..ab294351 --- /dev/null +++ b/cluster/hasql/cluster.go @@ -0,0 +1,235 @@ +package sql + +import ( + "context" + "database/sql" + "reflect" + "unsafe" + + "golang.yandex/hasql/v2" +) + +func newSQLRowError() *sql.Row { + row := &sql.Row{} + t := reflect.TypeOf(row).Elem() + field, _ := t.FieldByName("err") + rowPtr := unsafe.Pointer(row) + errFieldPtr := unsafe.Pointer(uintptr(rowPtr) + field.Offset) + errPtr := (*error)(errFieldPtr) + *errPtr = ErrorNoAliveNodes + return row +} + +type ClusterQuerier interface { + Querier + WaitForNodes(ctx context.Context, criterion ...hasql.NodeStateCriterion) error +} + +type Cluster struct { + hasql *hasql.Cluster[Querier] + options ClusterOptions +} + +// NewCluster returns [Querier] that provides cluster of nodes +func NewCluster[T Querier](opts ...ClusterOption) (ClusterQuerier, error) { + options := ClusterOptions{Context: context.Background()} + for _, opt := range opts { + opt(&options) + } + if options.NodeChecker == nil { + return nil, ErrClusterChecker + } + if options.NodeDiscoverer == nil { + return nil, ErrClusterDiscoverer + } + if options.NodePicker == nil { + return nil, ErrClusterPicker + } + + if options.Retries < 1 { + options.Retries = 1 + } + + if options.NodeStateCriterion == 0 { + options.NodeStateCriterion = hasql.Primary + } + + options.Options = append(options.Options, hasql.WithNodePicker(options.NodePicker)) + if p, ok := options.NodePicker.(*CustomPicker[Querier]); ok { + p.opts.Priority = options.NodePriority + } + + c, err := hasql.NewCluster( + options.NodeDiscoverer, + options.NodeChecker, + options.Options..., + ) + if err != nil { + return nil, err + } + + return &Cluster{hasql: c, options: options}, nil +} + +func (c *Cluster) BeginTx(ctx context.Context, opts *sql.TxOptions) (*sql.Tx, error) { + var tx *sql.Tx + var err error + + retries := 0 + c.hasql.NodesIter(c.getNodeStateCriterion(ctx))(func(n *hasql.Node[Querier]) bool { + for ; retries < c.options.Retries; retries++ { + if tx, err = n.DB().BeginTx(ctx, opts); err != nil && retries >= c.options.Retries { + return true + } + } + return false + }) + + if tx == nil && err == nil { + err = ErrorNoAliveNodes + } + + return tx, err +} + +func (c *Cluster) Close() error { + return c.hasql.Close() +} + +func (c *Cluster) Conn(ctx context.Context) (*sql.Conn, error) { + var conn *sql.Conn + var err error + + retries := 0 + c.hasql.NodesIter(c.getNodeStateCriterion(ctx))(func(n *hasql.Node[Querier]) bool { + for ; retries < c.options.Retries; retries++ { + if conn, err = n.DB().Conn(ctx); err != nil && retries >= c.options.Retries { + return true + } + } + return false + }) + + if conn == nil && err == nil { + err = ErrorNoAliveNodes + } + + return conn, err +} + +func (c *Cluster) ExecContext(ctx context.Context, query string, args ...interface{}) (sql.Result, error) { + var res sql.Result + var err error + + retries := 0 + c.hasql.NodesIter(c.getNodeStateCriterion(ctx))(func(n *hasql.Node[Querier]) bool { + for ; retries < c.options.Retries; retries++ { + if res, err = n.DB().ExecContext(ctx, query, args...); err != nil && retries >= c.options.Retries { + return true + } + } + return false + }) + + if res == nil && err == nil { + err = ErrorNoAliveNodes + } + + return res, err +} + +func (c *Cluster) PrepareContext(ctx context.Context, query string) (*sql.Stmt, error) { + var res *sql.Stmt + var err error + + retries := 0 + c.hasql.NodesIter(c.getNodeStateCriterion(ctx))(func(n *hasql.Node[Querier]) bool { + for ; retries < c.options.Retries; retries++ { + if res, err = n.DB().PrepareContext(ctx, query); err != nil && retries >= c.options.Retries { + return true + } + } + return false + }) + + if res == nil && err == nil { + err = ErrorNoAliveNodes + } + + return res, err +} + +func (c *Cluster) QueryContext(ctx context.Context, query string, args ...interface{}) (*sql.Rows, error) { + var res *sql.Rows + var err error + + retries := 0 + c.hasql.NodesIter(c.getNodeStateCriterion(ctx))(func(n *hasql.Node[Querier]) bool { + for ; retries < c.options.Retries; retries++ { + if res, err = n.DB().QueryContext(ctx, query); err != nil && err != sql.ErrNoRows && retries >= c.options.Retries { + return true + } + } + return false + }) + + if res == nil && err == nil { + err = ErrorNoAliveNodes + } + + return res, err +} + +func (c *Cluster) QueryRowContext(ctx context.Context, query string, args ...interface{}) *sql.Row { + var res *sql.Row + + retries := 0 + c.hasql.NodesIter(c.getNodeStateCriterion(ctx))(func(n *hasql.Node[Querier]) bool { + for ; retries < c.options.Retries; retries++ { + res = n.DB().QueryRowContext(ctx, query, args...) + if res.Err() == nil { + return false + } else if res.Err() != nil && retries >= c.options.Retries { + return false + } + } + return true + }) + + if res == nil { + res = newSQLRowError() + } + + return res +} + +func (c *Cluster) PingContext(ctx context.Context) error { + var err error + var ok bool + + retries := 0 + c.hasql.NodesIter(c.getNodeStateCriterion(ctx))(func(n *hasql.Node[Querier]) bool { + ok = true + for ; retries < c.options.Retries; retries++ { + if err = n.DB().PingContext(ctx); err != nil && retries >= c.options.Retries { + return true + } + } + return false + }) + + if !ok { + err = ErrorNoAliveNodes + } + + return err +} + +func (c *Cluster) WaitForNodes(ctx context.Context, criterions ...hasql.NodeStateCriterion) error { + for _, criterion := range criterions { + if _, err := c.hasql.WaitForNode(ctx, criterion); err != nil { + return err + } + } + return nil +} diff --git a/cluster/hasql/cluster_test.go b/cluster/hasql/cluster_test.go new file mode 100644 index 00000000..6a1455ba --- /dev/null +++ b/cluster/hasql/cluster_test.go @@ -0,0 +1,171 @@ +package sql + +import ( + "context" + "fmt" + "testing" + "time" + + "github.com/DATA-DOG/go-sqlmock" + "golang.yandex/hasql/v2" +) + +func TestNewCluster(t *testing.T) { + dbMaster, dbMasterMock, err := sqlmock.New(sqlmock.MonitorPingsOption(true)) + if err != nil { + t.Fatal(err) + } + defer dbMaster.Close() + dbMasterMock.MatchExpectationsInOrder(false) + + dbMasterMock.ExpectQuery(`.*pg_is_in_recovery.*`).WillReturnRows( + sqlmock.NewRowsWithColumnDefinition( + sqlmock.NewColumn("role").OfType("int8", 0), + sqlmock.NewColumn("replication_lag").OfType("int8", 0)). + AddRow(1, 0)). + RowsWillBeClosed(). + WithoutArgs() + + dbMasterMock.ExpectQuery(`SELECT node_name as name`).WillReturnRows( + sqlmock.NewRows([]string{"name"}). + AddRow("master-dc1")) + + dbDRMaster, dbDRMasterMock, err := sqlmock.New(sqlmock.MonitorPingsOption(true)) + if err != nil { + t.Fatal(err) + } + defer dbDRMaster.Close() + dbDRMasterMock.MatchExpectationsInOrder(false) + + dbDRMasterMock.ExpectQuery(`.*pg_is_in_recovery.*`).WillReturnRows( + sqlmock.NewRowsWithColumnDefinition( + sqlmock.NewColumn("role").OfType("int8", 0), + sqlmock.NewColumn("replication_lag").OfType("int8", 0)). + AddRow(2, 40)). + RowsWillBeClosed(). + WithoutArgs() + + dbDRMasterMock.ExpectQuery(`SELECT node_name as name`).WillReturnRows( + sqlmock.NewRows([]string{"name"}). + AddRow("drmaster1-dc2")) + + dbDRMasterMock.ExpectQuery(`SELECT node_name as name`).WillReturnRows( + sqlmock.NewRows([]string{"name"}). + AddRow("drmaster")) + + dbSlaveDC1, dbSlaveDC1Mock, err := sqlmock.New(sqlmock.MonitorPingsOption(true)) + if err != nil { + t.Fatal(err) + } + defer dbSlaveDC1.Close() + dbSlaveDC1Mock.MatchExpectationsInOrder(false) + + dbSlaveDC1Mock.ExpectQuery(`.*pg_is_in_recovery.*`).WillReturnRows( + sqlmock.NewRowsWithColumnDefinition( + sqlmock.NewColumn("role").OfType("int8", 0), + sqlmock.NewColumn("replication_lag").OfType("int8", 0)). + AddRow(2, 50)). + RowsWillBeClosed(). + WithoutArgs() + + dbSlaveDC1Mock.ExpectQuery(`SELECT node_name as name`).WillReturnRows( + sqlmock.NewRows([]string{"name"}). + AddRow("slave-dc1")) + + dbSlaveDC2, dbSlaveDC2Mock, err := sqlmock.New(sqlmock.MonitorPingsOption(true)) + if err != nil { + t.Fatal(err) + } + defer dbSlaveDC2.Close() + dbSlaveDC1Mock.MatchExpectationsInOrder(false) + + dbSlaveDC2Mock.ExpectQuery(`.*pg_is_in_recovery.*`).WillReturnRows( + sqlmock.NewRowsWithColumnDefinition( + sqlmock.NewColumn("role").OfType("int8", 0), + sqlmock.NewColumn("replication_lag").OfType("int8", 0)). + AddRow(2, 50)). + RowsWillBeClosed(). + WithoutArgs() + + dbSlaveDC2Mock.ExpectQuery(`SELECT node_name as name`).WillReturnRows( + sqlmock.NewRows([]string{"name"}). + AddRow("slave-dc1")) + + tctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) + defer cancel() + + c, err := NewCluster[Querier]( + WithClusterContext(tctx), + WithClusterNodeChecker(hasql.PostgreSQLChecker), + WithClusterNodePicker(NewCustomPicker[Querier]( + CustomPickerMaxLag(100), + )), + WithClusterNodes( + ClusterNode{"slave-dc1", dbSlaveDC1, 1}, + ClusterNode{"master-dc1", dbMaster, 1}, + ClusterNode{"slave-dc2", dbSlaveDC2, 2}, + ClusterNode{"drmaster1-dc2", dbDRMaster, 0}, + ), + WithClusterOptions( + hasql.WithUpdateInterval[Querier](2*time.Second), + hasql.WithUpdateTimeout[Querier](1*time.Second), + ), + ) + if err != nil { + t.Fatal(err) + } + defer c.Close() + + if err = c.WaitForNodes(tctx, hasql.Primary, hasql.Standby); err != nil { + t.Fatal(err) + } + + time.Sleep(500 * time.Millisecond) + + node1Name := "" + fmt.Printf("check for Standby\n") + if row := c.QueryRowContext(NodeStateCriterion(tctx, hasql.Standby), "SELECT node_name as name"); row.Err() != nil { + t.Fatal(row.Err()) + } else if err = row.Scan(&node1Name); err != nil { + t.Fatal(err) + } else if "slave-dc1" != node1Name { + t.Fatalf("invalid node name %s != %s", "slave-dc1", node1Name) + } + + dbSlaveDC1Mock.ExpectQuery(`SELECT node_name as name`).WillReturnRows( + sqlmock.NewRows([]string{"name"}). + AddRow("slave-dc1")) + + node2Name := "" + fmt.Printf("check for PreferStandby\n") + if row := c.QueryRowContext(NodeStateCriterion(tctx, hasql.PreferStandby), "SELECT node_name as name"); row.Err() != nil { + t.Fatal(row.Err()) + } else if err = row.Scan(&node2Name); err != nil { + t.Fatal(err) + } else if "slave-dc1" != node2Name { + t.Fatalf("invalid node name %s != %s", "slave-dc1", node2Name) + } + + node3Name := "" + fmt.Printf("check for PreferPrimary\n") + if row := c.QueryRowContext(NodeStateCriterion(tctx, hasql.PreferPrimary), "SELECT node_name as name"); row.Err() != nil { + t.Fatal(row.Err()) + } else if err = row.Scan(&node3Name); err != nil { + t.Fatal(err) + } else if "master-dc1" != node3Name { + t.Fatalf("invalid node name %s != %s", "master-dc1", node3Name) + } + + dbSlaveDC1Mock.ExpectQuery(`.*`).WillReturnRows(sqlmock.NewRows([]string{"role"}).RowError(1, fmt.Errorf("row error"))) + + time.Sleep(2 * time.Second) + + fmt.Printf("check for PreferStandby\n") + if row := c.QueryRowContext(NodeStateCriterion(tctx, hasql.PreferStandby), "SELECT node_name as name"); row.Err() == nil { + t.Fatal("must return error") + } + + if dbMasterErr := dbMasterMock.ExpectationsWereMet(); dbMasterErr != nil { + t.Error(dbMasterErr) + } +} diff --git a/cluster/hasql/db.go b/cluster/hasql/db.go new file mode 100644 index 00000000..575e8158 --- /dev/null +++ b/cluster/hasql/db.go @@ -0,0 +1,25 @@ +package sql + +import ( + "context" + "database/sql" +) + +type Querier interface { + // Basic connection methods + PingContext(ctx context.Context) error + Close() error + + // Query methods with context + ExecContext(ctx context.Context, query string, args ...interface{}) (sql.Result, error) + QueryContext(ctx context.Context, query string, args ...interface{}) (*sql.Rows, error) + QueryRowContext(ctx context.Context, query string, args ...interface{}) *sql.Row + + // Prepared statements with context + PrepareContext(ctx context.Context, query string) (*sql.Stmt, error) + + // Transaction management with context + BeginTx(ctx context.Context, opts *sql.TxOptions) (*sql.Tx, error) + + Conn(ctx context.Context) (*sql.Conn, error) +} diff --git a/cluster/hasql/driver.go b/cluster/hasql/driver.go new file mode 100644 index 00000000..e04def6c --- /dev/null +++ b/cluster/hasql/driver.go @@ -0,0 +1,295 @@ +package sql + +import ( + "context" + "database/sql" + "database/sql/driver" + "io" + "sync" + "time" +) + +// OpenDBWithCluster creates a [*sql.DB] that uses the [ClusterQuerier] +func OpenDBWithCluster(db ClusterQuerier) (*sql.DB, error) { + driver := NewClusterDriver(db) + connector, err := driver.OpenConnector("") + if err != nil { + return nil, err + } + return sql.OpenDB(connector), nil +} + +// ClusterDriver implements [driver.Driver] and driver.Connector for an existing [Querier] +type ClusterDriver struct { + db ClusterQuerier +} + +// NewClusterDriver creates a new [driver.Driver] that uses an existing [ClusterQuerier] +func NewClusterDriver(db ClusterQuerier) *ClusterDriver { + return &ClusterDriver{db: db} +} + +// Open implements [driver.Driver.Open] +func (d *ClusterDriver) Open(name string) (driver.Conn, error) { + return d.Connect(context.Background()) +} + +// OpenConnector implements [driver.DriverContext.OpenConnector] +func (d *ClusterDriver) OpenConnector(name string) (driver.Connector, error) { + return d, nil +} + +// Connect implements [driver.Connector.Connect] +func (d *ClusterDriver) Connect(ctx context.Context) (driver.Conn, error) { + conn, err := d.db.Conn(ctx) + if err != nil { + return nil, err + } + return &dbConn{conn: conn}, nil +} + +// Driver implements [driver.Connector.Driver] +func (d *ClusterDriver) Driver() driver.Driver { + return d +} + +// dbConn implements driver.Conn with both context and legacy methods +type dbConn struct { + conn *sql.Conn + mu sync.Mutex +} + +// Prepare implements [driver.Conn.Prepare] (legacy method) +func (c *dbConn) Prepare(query string) (driver.Stmt, error) { + return c.PrepareContext(context.Background(), query) +} + +// PrepareContext implements [driver.ConnPrepareContext.PrepareContext] +func (c *dbConn) PrepareContext(ctx context.Context, query string) (driver.Stmt, error) { + c.mu.Lock() + defer c.mu.Unlock() + + stmt, err := c.conn.PrepareContext(ctx, query) + if err != nil { + return nil, err + } + + return &dbStmt{stmt: stmt}, nil +} + +// Exec implements [driver.Execer.Exec] (legacy method) +func (c *dbConn) Exec(query string, args []driver.Value) (driver.Result, error) { + namedArgs := make([]driver.NamedValue, len(args)) + for i, value := range args { + namedArgs[i] = driver.NamedValue{Value: value} + } + return c.ExecContext(context.Background(), query, namedArgs) +} + +// ExecContext implements [driver.ExecerContext.ExecContext] +func (c *dbConn) ExecContext(ctx context.Context, query string, args []driver.NamedValue) (driver.Result, error) { + c.mu.Lock() + defer c.mu.Unlock() + + // Convert driver.NamedValue to any + interfaceArgs := make([]any, len(args)) + for i, arg := range args { + interfaceArgs[i] = arg.Value + } + + return c.conn.ExecContext(ctx, query, interfaceArgs...) +} + +// Query implements [driver.Queryer.Query] (legacy method) +func (c *dbConn) Query(query string, args []driver.Value) (driver.Rows, error) { + namedArgs := make([]driver.NamedValue, len(args)) + for i, value := range args { + namedArgs[i] = driver.NamedValue{Value: value} + } + return c.QueryContext(context.Background(), query, namedArgs) +} + +// QueryContext implements [driver.QueryerContext.QueryContext] +func (c *dbConn) QueryContext(ctx context.Context, query string, args []driver.NamedValue) (driver.Rows, error) { + c.mu.Lock() + defer c.mu.Unlock() + + // Convert driver.NamedValue to any + interfaceArgs := make([]any, len(args)) + for i, arg := range args { + interfaceArgs[i] = arg.Value + } + + rows, err := c.conn.QueryContext(ctx, query, interfaceArgs...) + if err != nil { + return nil, err + } + + return &dbRows{rows: rows}, nil +} + +// Begin implements [driver.Conn.Begin] (legacy method) +func (c *dbConn) Begin() (driver.Tx, error) { + return c.BeginTx(context.Background(), driver.TxOptions{}) +} + +// BeginTx implements [driver.ConnBeginTx.BeginTx] +func (c *dbConn) BeginTx(ctx context.Context, opts driver.TxOptions) (driver.Tx, error) { + c.mu.Lock() + defer c.mu.Unlock() + + sqlOpts := &sql.TxOptions{ + Isolation: sql.IsolationLevel(opts.Isolation), + ReadOnly: opts.ReadOnly, + } + + tx, err := c.conn.BeginTx(ctx, sqlOpts) + if err != nil { + return nil, err + } + + return &dbTx{tx: tx}, nil +} + +// Ping implements [driver.Pinger.Ping] +func (c *dbConn) Ping(ctx context.Context) error { + return c.conn.PingContext(ctx) +} + +// Close implements [driver.Conn.Close] +func (c *dbConn) Close() error { + return c.conn.Close() +} + +// IsValid implements [driver.Validator.IsValid] +func (c *dbConn) IsValid() bool { + // Ping with a short timeout to check if the connection is still valid + ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second) + defer cancel() + + return c.conn.PingContext(ctx) == nil +} + +// dbStmt implements [driver.Stmt] with both context and legacy methods +type dbStmt struct { + stmt *sql.Stmt + mu sync.Mutex +} + +// Close implements [driver.Stmt.Close] +func (s *dbStmt) Close() error { + s.mu.Lock() + defer s.mu.Unlock() + return s.stmt.Close() +} + +// Close implements [driver.Stmt.NumInput] +func (s *dbStmt) NumInput() int { + return -1 // Number of parameters is unknown +} + +// Exec implements [driver.Stmt.Exec] (legacy method) +func (s *dbStmt) Exec(args []driver.Value) (driver.Result, error) { + namedArgs := make([]driver.NamedValue, len(args)) + for i, value := range args { + namedArgs[i] = driver.NamedValue{Value: value} + } + return s.ExecContext(context.Background(), namedArgs) +} + +// ExecContext implements [driver.StmtExecContext.ExecContext] +func (s *dbStmt) ExecContext(ctx context.Context, args []driver.NamedValue) (driver.Result, error) { + s.mu.Lock() + defer s.mu.Unlock() + + interfaceArgs := make([]any, len(args)) + for i, arg := range args { + interfaceArgs[i] = arg.Value + } + return s.stmt.ExecContext(ctx, interfaceArgs...) +} + +// Query implements [driver.Stmt.Query] (legacy method) +func (s *dbStmt) Query(args []driver.Value) (driver.Rows, error) { + namedArgs := make([]driver.NamedValue, len(args)) + for i, value := range args { + namedArgs[i] = driver.NamedValue{Value: value} + } + return s.QueryContext(context.Background(), namedArgs) +} + +// QueryContext implements [driver.StmtQueryContext.QueryContext] +func (s *dbStmt) QueryContext(ctx context.Context, args []driver.NamedValue) (driver.Rows, error) { + s.mu.Lock() + defer s.mu.Unlock() + + interfaceArgs := make([]any, len(args)) + for i, arg := range args { + interfaceArgs[i] = arg.Value + } + + rows, err := s.stmt.QueryContext(ctx, interfaceArgs...) + if err != nil { + return nil, err + } + + return &dbRows{rows: rows}, nil +} + +// dbRows implements [driver.Rows] +type dbRows struct { + rows *sql.Rows +} + +// Columns implements [driver.Rows.Columns] +func (r *dbRows) Columns() []string { + cols, err := r.rows.Columns() + if err != nil { + // This shouldn't happen if the query was successful + return []string{} + } + return cols +} + +// Close implements [driver.Rows.Close] +func (r *dbRows) Close() error { + return r.rows.Close() +} + +// Next implements [driver.Rows.Next] +func (r *dbRows) Next(dest []driver.Value) error { + if !r.rows.Next() { + if err := r.rows.Err(); err != nil { + return err + } + return io.EOF + } + + // Create a slice of interfaces to scan into + scanArgs := make([]any, len(dest)) + for i := range scanArgs { + scanArgs[i] = &dest[i] + } + + return r.rows.Scan(scanArgs...) +} + +// dbTx implements [driver.Tx] +type dbTx struct { + tx *sql.Tx + mu sync.Mutex +} + +// Commit implements [driver.Tx.Commit] +func (t *dbTx) Commit() error { + t.mu.Lock() + defer t.mu.Unlock() + return t.tx.Commit() +} + +// Rollback implements [driver.Tx.Rollback] +func (t *dbTx) Rollback() error { + t.mu.Lock() + defer t.mu.Unlock() + return t.tx.Rollback() +} diff --git a/cluster/hasql/driver_test.go b/cluster/hasql/driver_test.go new file mode 100644 index 00000000..17f357a4 --- /dev/null +++ b/cluster/hasql/driver_test.go @@ -0,0 +1,141 @@ +package sql + +import ( + "context" + "testing" + "time" + + "github.com/DATA-DOG/go-sqlmock" + "golang.yandex/hasql/v2" +) + +func TestDriver(t *testing.T) { + dbMaster, dbMasterMock, err := sqlmock.New(sqlmock.MonitorPingsOption(true)) + if err != nil { + t.Fatal(err) + } + defer dbMaster.Close() + dbMasterMock.MatchExpectationsInOrder(false) + + dbMasterMock.ExpectQuery(`.*pg_is_in_recovery.*`).WillReturnRows( + sqlmock.NewRowsWithColumnDefinition( + sqlmock.NewColumn("role").OfType("int8", 0), + sqlmock.NewColumn("replication_lag").OfType("int8", 0)). + AddRow(1, 0)). + RowsWillBeClosed(). + WithoutArgs() + + dbMasterMock.ExpectQuery(`SELECT node_name as name`).WillReturnRows( + sqlmock.NewRows([]string{"name"}). + AddRow("master-dc1")) + + dbDRMaster, dbDRMasterMock, err := sqlmock.New(sqlmock.MonitorPingsOption(true)) + if err != nil { + t.Fatal(err) + } + defer dbDRMaster.Close() + dbDRMasterMock.MatchExpectationsInOrder(false) + + dbDRMasterMock.ExpectQuery(`.*pg_is_in_recovery.*`).WillReturnRows( + sqlmock.NewRowsWithColumnDefinition( + sqlmock.NewColumn("role").OfType("int8", 0), + sqlmock.NewColumn("replication_lag").OfType("int8", 0)). + AddRow(2, 40)). + RowsWillBeClosed(). + WithoutArgs() + + dbDRMasterMock.ExpectQuery(`SELECT node_name as name`).WillReturnRows( + sqlmock.NewRows([]string{"name"}). + AddRow("drmaster1-dc2")) + + dbDRMasterMock.ExpectQuery(`SELECT node_name as name`).WillReturnRows( + sqlmock.NewRows([]string{"name"}). + AddRow("drmaster")) + + dbSlaveDC1, dbSlaveDC1Mock, err := sqlmock.New(sqlmock.MonitorPingsOption(true)) + if err != nil { + t.Fatal(err) + } + defer dbSlaveDC1.Close() + dbSlaveDC1Mock.MatchExpectationsInOrder(false) + + dbSlaveDC1Mock.ExpectQuery(`.*pg_is_in_recovery.*`).WillReturnRows( + sqlmock.NewRowsWithColumnDefinition( + sqlmock.NewColumn("role").OfType("int8", 0), + sqlmock.NewColumn("replication_lag").OfType("int8", 0)). + AddRow(2, 50)). + RowsWillBeClosed(). + WithoutArgs() + + dbSlaveDC1Mock.ExpectQuery(`SELECT node_name as name`).WillReturnRows( + sqlmock.NewRows([]string{"name"}). + AddRow("slave-dc1")) + + dbSlaveDC2, dbSlaveDC2Mock, err := sqlmock.New(sqlmock.MonitorPingsOption(true)) + if err != nil { + t.Fatal(err) + } + defer dbSlaveDC2.Close() + dbSlaveDC1Mock.MatchExpectationsInOrder(false) + + dbSlaveDC2Mock.ExpectQuery(`.*pg_is_in_recovery.*`).WillReturnRows( + sqlmock.NewRowsWithColumnDefinition( + sqlmock.NewColumn("role").OfType("int8", 0), + sqlmock.NewColumn("replication_lag").OfType("int8", 0)). + AddRow(2, 50)). + RowsWillBeClosed(). + WithoutArgs() + + dbSlaveDC2Mock.ExpectQuery(`SELECT node_name as name`).WillReturnRows( + sqlmock.NewRows([]string{"name"}). + AddRow("slave-dc1")) + + tctx, cancel := context.WithTimeout(t.Context(), 10*time.Second) + defer cancel() + + c, err := NewCluster[Querier]( + WithClusterContext(tctx), + WithClusterNodeChecker(hasql.PostgreSQLChecker), + WithClusterNodePicker(NewCustomPicker[Querier]( + CustomPickerMaxLag(100), + )), + WithClusterNodes( + ClusterNode{"slave-dc1", dbSlaveDC1, 1}, + ClusterNode{"master-dc1", dbMaster, 1}, + ClusterNode{"slave-dc2", dbSlaveDC2, 2}, + ClusterNode{"drmaster1-dc2", dbDRMaster, 0}, + ), + WithClusterOptions( + hasql.WithUpdateInterval[Querier](2*time.Second), + hasql.WithUpdateTimeout[Querier](1*time.Second), + ), + ) + if err != nil { + t.Fatal(err) + } + defer c.Close() + + if err = c.WaitForNodes(tctx, hasql.Primary, hasql.Standby); err != nil { + t.Fatal(err) + } + + db, err := OpenDBWithCluster(c) + if err != nil { + t.Fatal(err) + } + + // Use context methods + row := db.QueryRowContext(NodeStateCriterion(t.Context(), hasql.Primary), "SELECT node_name as name") + if err = row.Err(); err != nil { + t.Fatal(err) + } + + nodeName := "" + if err = row.Scan(&nodeName); err != nil { + t.Fatal(err) + } + + if nodeName != "master-dc1" { + t.Fatalf("invalid node_name %s != %s", "master-dc1", nodeName) + } +} diff --git a/cluster/hasql/error.go b/cluster/hasql/error.go new file mode 100644 index 00000000..c92fd0ee --- /dev/null +++ b/cluster/hasql/error.go @@ -0,0 +1,10 @@ +package sql + +import "errors" + +var ( + ErrClusterChecker = errors.New("cluster node checker required") + ErrClusterDiscoverer = errors.New("cluster node discoverer required") + ErrClusterPicker = errors.New("cluster node picker required") + ErrorNoAliveNodes = errors.New("cluster no alive nodes") +) diff --git a/cluster/hasql/options.go b/cluster/hasql/options.go new file mode 100644 index 00000000..6226a38a --- /dev/null +++ b/cluster/hasql/options.go @@ -0,0 +1,110 @@ +package sql + +import ( + "context" + "math" + + "golang.yandex/hasql/v2" +) + +// ClusterOptions contains cluster specific options +type ClusterOptions struct { + NodeChecker hasql.NodeChecker + NodePicker hasql.NodePicker[Querier] + NodeDiscoverer hasql.NodeDiscoverer[Querier] + Options []hasql.ClusterOpt[Querier] + Context context.Context + Retries int + NodePriority map[string]int32 + NodeStateCriterion hasql.NodeStateCriterion +} + +// ClusterOption apply cluster options to ClusterOptions +type ClusterOption func(*ClusterOptions) + +// WithClusterNodeChecker pass hasql.NodeChecker to cluster options +func WithClusterNodeChecker(c hasql.NodeChecker) ClusterOption { + return func(o *ClusterOptions) { + o.NodeChecker = c + } +} + +// WithClusterNodePicker pass hasql.NodePicker to cluster options +func WithClusterNodePicker(p hasql.NodePicker[Querier]) ClusterOption { + return func(o *ClusterOptions) { + o.NodePicker = p + } +} + +// WithClusterNodeDiscoverer pass hasql.NodeDiscoverer to cluster options +func WithClusterNodeDiscoverer(d hasql.NodeDiscoverer[Querier]) ClusterOption { + return func(o *ClusterOptions) { + o.NodeDiscoverer = d + } +} + +// WithRetries retry count on other nodes in case of error +func WithRetries(n int) ClusterOption { + return func(o *ClusterOptions) { + o.Retries = n + } +} + +// WithClusterContext pass context.Context to cluster options and used for checks +func WithClusterContext(ctx context.Context) ClusterOption { + return func(o *ClusterOptions) { + o.Context = ctx + } +} + +// WithClusterOptions pass hasql.ClusterOpt +func WithClusterOptions(opts ...hasql.ClusterOpt[Querier]) ClusterOption { + return func(o *ClusterOptions) { + o.Options = append(o.Options, opts...) + } +} + +// WithClusterNodeStateCriterion pass default hasql.NodeStateCriterion +func WithClusterNodeStateCriterion(c hasql.NodeStateCriterion) ClusterOption { + return func(o *ClusterOptions) { + o.NodeStateCriterion = c + } +} + +type ClusterNode struct { + Name string + DB Querier + Priority int32 +} + +// WithClusterNodes create cluster with static NodeDiscoverer +func WithClusterNodes(cns ...ClusterNode) ClusterOption { + return func(o *ClusterOptions) { + nodes := make([]*hasql.Node[Querier], 0, len(cns)) + if o.NodePriority == nil { + o.NodePriority = make(map[string]int32, len(cns)) + } + for _, cn := range cns { + nodes = append(nodes, hasql.NewNode(cn.Name, cn.DB)) + if cn.Priority == 0 { + cn.Priority = math.MaxInt32 + } + o.NodePriority[cn.Name] = cn.Priority + } + o.NodeDiscoverer = hasql.NewStaticNodeDiscoverer(nodes...) + } +} + +type nodeStateCriterionKey struct{} + +// NodeStateCriterion inject hasql.NodeStateCriterion to context +func NodeStateCriterion(ctx context.Context, c hasql.NodeStateCriterion) context.Context { + return context.WithValue(ctx, nodeStateCriterionKey{}, c) +} + +func (c *Cluster) getNodeStateCriterion(ctx context.Context) hasql.NodeStateCriterion { + if v, ok := ctx.Value(nodeStateCriterionKey{}).(hasql.NodeStateCriterion); ok { + return v + } + return c.options.NodeStateCriterion +} diff --git a/cluster/hasql/picker.go b/cluster/hasql/picker.go new file mode 100644 index 00000000..5e131fca --- /dev/null +++ b/cluster/hasql/picker.go @@ -0,0 +1,113 @@ +package sql + +import ( + "fmt" + "math" + "time" + + "golang.yandex/hasql/v2" +) + +// compile time guard +var _ hasql.NodePicker[Querier] = (*CustomPicker[Querier])(nil) + +// CustomPickerOptions holds options to pick nodes +type CustomPickerOptions struct { + MaxLag int + Priority map[string]int32 + Retries int +} + +// CustomPickerOption func apply option to CustomPickerOptions +type CustomPickerOption func(*CustomPickerOptions) + +// CustomPickerMaxLag specifies max lag for which node can be used +func CustomPickerMaxLag(n int) CustomPickerOption { + return func(o *CustomPickerOptions) { + o.MaxLag = n + } +} + +// NewCustomPicker creates new node picker +func NewCustomPicker[T Querier](opts ...CustomPickerOption) *CustomPicker[Querier] { + options := CustomPickerOptions{} + for _, o := range opts { + o(&options) + } + return &CustomPicker[Querier]{opts: options} +} + +// CustomPicker holds node picker options +type CustomPicker[T Querier] struct { + opts CustomPickerOptions +} + +// PickNode used to return specific node +func (p *CustomPicker[T]) PickNode(cnodes []hasql.CheckedNode[T]) hasql.CheckedNode[T] { + for _, n := range cnodes { + fmt.Printf("node %s\n", n.Node.String()) + } + return cnodes[0] +} + +func (p *CustomPicker[T]) getPriority(nodeName string) int32 { + if prio, ok := p.opts.Priority[nodeName]; ok { + return prio + } + return math.MaxInt32 // Default to lowest priority +} + +// CompareNodes used to sort nodes +func (p *CustomPicker[T]) CompareNodes(a, b hasql.CheckedNode[T]) int { + // Get replication lag values + aLag := a.Info.(interface{ ReplicationLag() int }).ReplicationLag() + bLag := b.Info.(interface{ ReplicationLag() int }).ReplicationLag() + + // First check that lag lower then MaxLag + if aLag > p.opts.MaxLag && bLag > p.opts.MaxLag { + return 0 // both are equal + } + + // If one node exceeds MaxLag and the other doesn't, prefer the one that doesn't + if aLag > p.opts.MaxLag { + return 1 // b is better + } + if bLag > p.opts.MaxLag { + return -1 // a is better + } + + // Get node priorities + aPrio := p.getPriority(a.Node.String()) + bPrio := p.getPriority(b.Node.String()) + + // if both priority equals + if aPrio == bPrio { + // First compare by replication lag + if aLag < bLag { + return -1 + } + if aLag > bLag { + return 1 + } + // If replication lag is equal, compare by latency + aLatency := a.Info.(interface{ Latency() time.Duration }).Latency() + bLatency := b.Info.(interface{ Latency() time.Duration }).Latency() + + if aLatency < bLatency { + return -1 + } + if aLatency > bLatency { + return 1 + } + + // If lag and latency is equal + return 0 + } + + // If priorities are different, prefer the node with lower priority value + if aPrio < bPrio { + return -1 + } + + return 1 +} -- 2.49.1 From e09c34c6917fd0351431e1f0aa4a57246037b3e6 Mon Sep 17 00:00:00 2001 From: Vasiliy Tolstov Date: Sat, 20 Sep 2025 23:00:57 +0300 Subject: [PATCH 5/6] split files Signed-off-by: Vasiliy Tolstov --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 754587ed..cca89a9b 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,5 @@ # Micro -![Coverage](https://img.shields.io/badge/Coverage-34.0%25-yellow) +![Coverage](https://img.shields.io/badge/Coverage-33.7%25-yellow) [![License](https://img.shields.io/:license-apache-blue.svg)](https://opensource.org/licenses/Apache-2.0) [![Doc](https://img.shields.io/badge/go.dev-reference-007d9c?logo=go&logoColor=white&style=flat-square)](https://pkg.go.dev/go.unistack.org/micro/v3?tab=overview) [![Status](https://git.unistack.org/unistack-org/micro/actions/workflows/job_tests.yml/badge.svg?branch=v3)](https://git.unistack.org/unistack-org/micro/actions?query=workflow%3Abuild+branch%3Av3+event%3Apush) -- 2.49.1 From c83d9414903fa6d84763da25cc948a41e418c3c5 Mon Sep 17 00:00:00 2001 From: Vasiliy Tolstov Date: Sat, 20 Sep 2025 23:51:02 +0300 Subject: [PATCH 6/6] split files Signed-off-by: Vasiliy Tolstov --- go.mod | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 36be164c..c2389f79 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module go.unistack.org/micro/v3 -go 1.23.0 +go 1.24.0 require ( dario.cat/mergo v1.0.1 -- 2.49.1