Update workflows (#126)

Co-authored-by: Aleksandr Tolstikhin <atolstikhin@mtsbank.ru>
Co-authored-by: Василий Толстов <v.tolstov@unistack.org>
Reviewed-on: #126
Co-authored-by: Александр Толстихин <tolstihin1996@mail.ru>
Co-committed-by: Александр Толстихин <tolstihin1996@mail.ru>

.gitignore

@@ -0,0 +1 @@
+.idea

event.go

@@ -0,0 +1,86 @@
+package redis
+
+import (
+	"context"
+	"errors"
+	"net"
+	"time"
+
+	goredis "github.com/redis/go-redis/v9"
+	"go.unistack.org/micro/v3/store"
+)
+
+type eventHook struct {
+	s *Store
+}
+
+var _ goredis.Hook = (*eventHook)(nil)
+
+func newEventHook(s *Store) *eventHook {
+	return &eventHook{s: s}
+}
+
+func (h *eventHook) DialHook(hook goredis.DialHook) goredis.DialHook {
+	return func(ctx context.Context, network, addr string) (net.Conn, error) {
+		conn, err := hook(ctx, network, addr)
+		if err != nil {
+			if !isRedisError(err) {
+				if h.s.connected.CompareAndSwap(1, 0) {
+					h.s.sendEvent(&event{ts: time.Now(), err: err, t: store.EventTypeDisconnect})
+				}
+			} else {
+				h.s.connected.Store(1)
+			}
+		} else {
+			if h.s.connected.CompareAndSwap(0, 1) {
+				h.s.sendEvent(&event{ts: time.Now(), err: err, t: store.EventTypeConnect})
+			}
+		}
+		return conn, err
+	}
+}
+
+func (h *eventHook) ProcessHook(hook goredis.ProcessHook) goredis.ProcessHook {
+	return func(ctx context.Context, cmd goredis.Cmder) error {
+		err := hook(ctx, cmd)
+		if err != nil {
+			if !isRedisError(err) {
+				if h.s.connected.CompareAndSwap(1, 0) {
+					h.s.sendEvent(&event{ts: time.Now(), err: err, t: store.EventTypeDisconnect})
+				}
+			} else {
+				h.s.connected.Store(1)
+			}
+		} else {
+			if h.s.connected.CompareAndSwap(0, 1) {
+				h.s.sendEvent(&event{ts: time.Now(), err: err, t: store.EventTypeConnect})
+			}
+		}
+		return err
+	}
+}
+
+func (h *eventHook) ProcessPipelineHook(hook goredis.ProcessPipelineHook) goredis.ProcessPipelineHook {
+	return func(ctx context.Context, cmds []goredis.Cmder) error {
+		err := hook(ctx, cmds)
+		if err != nil {
+			if !isRedisError(err) {
+				if h.s.connected.CompareAndSwap(1, 0) {
+					h.s.sendEvent(&event{ts: time.Now(), err: err, t: store.EventTypeDisconnect})
+				}
+			} else {
+				h.s.connected.Store(1)
+			}
+		} else {
+			if h.s.connected.CompareAndSwap(0, 1) {
+				h.s.sendEvent(&event{ts: time.Now(), err: err, t: store.EventTypeConnect})
+			}
+		}
+		return err
+	}
+}
+
+func isRedisError(err error) bool {
+	var rerr goredis.Error
+	return errors.As(err, &rerr)
+}

go.mod

@@ -1,14 +1,19 @@
-module go.unistack.org/micro-store-redis/v4
+module go.unistack.org/micro-store-redis/v3
 
-go 1.20
+go 1.22
+
+toolchain go1.22.4
 
 require (
-	github.com/redis/go-redis/v9 v9.2.1
+	github.com/redis/go-redis/extra/rediscmd/v9 v9.7.0
-	go.unistack.org/micro/v4 v4.0.7
+	github.com/redis/go-redis/v9 v9.7.0
+	go.unistack.org/micro/v3 v3.10.108
 )
 
 require (
-	github.com/cespare/xxhash/v2 v2.2.0 // indirect
+	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
-	github.com/patrickmn/go-cache v2.1.0+incompatible // indirect
+	github.com/google/go-cmp v0.6.0 // indirect
+	go.unistack.org/micro-proto/v3 v3.4.1 // indirect
+	google.golang.org/protobuf v1.35.2 // indirect
 )

go.sum

@@ -1,12 +1,20 @@
 github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
+github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=
 github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
-github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
+github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0=
-github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
+github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
+github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
 github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
-github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
+github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
-github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
+github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
-github.com/redis/go-redis/v9 v9.2.1 h1:WlYJg71ODF0dVspZZCpYmoF1+U1Jjk9Rwd7pq6QmlCg=
+github.com/redis/go-redis/extra/rediscmd/v9 v9.7.0 h1:BIx9TNZH/Jsr4l1i7VVxnV0JPiwYj8qyrHyuL0fGZrk=
-github.com/redis/go-redis/v9 v9.2.1/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M=
+github.com/redis/go-redis/extra/rediscmd/v9 v9.7.0/go.mod h1:eTg/YQtGYAZD5r3DlGlJptJ45AHA+/G+2NPn30PKzik=
-go.unistack.org/micro/v4 v4.0.7 h1:2lwtZlHcSwgkahhFbkI4x1lOS79lw8uLHtcEhlFF+AM=
+github.com/redis/go-redis/v9 v9.7.0 h1:HhLSs+B6O021gwzl+locl0zEDnyNkxMtf/Z3NNBMa9E=
-go.unistack.org/micro/v4 v4.0.7/go.mod h1:bVEYTlPi0EsdgZZt311bIroDg9ict7ky3C87dSCCAGk=
+github.com/redis/go-redis/v9 v9.7.0/go.mod h1:f6zhXITC7JUJIlPEiBOTXxJgPLdZcA93GewI7inzyWw=
+go.unistack.org/micro-proto/v3 v3.4.1 h1:UTjLSRz2YZuaHk9iSlVqqsA50JQNAEK2ZFboGqtEa9Q=
+go.unistack.org/micro-proto/v3 v3.4.1/go.mod h1:okx/cnOhzuCX0ggl/vToatbCupi0O44diiiLLsZ93Zo=
+go.unistack.org/micro/v3 v3.10.108 h1:3L7SkilMVLtH8y3pQIPtr3jjQYrf0AMv1oAkoL3nFkE=
+go.unistack.org/micro/v3 v3.10.108/go.mod h1:YzMldzHN9Ei+zy5t/Psu7RUWDZwUfrNYiStSQtTz90g=
+google.golang.org/protobuf v1.35.2 h1:8Ar7bF+apOIoThw1EdZl0p1oWvMqTHmpA2fRTyZO8io=
+google.golang.org/protobuf v1.35.2/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE=

options.go

@@ -1,12 +1,67 @@
 package redis
 
 import (
-	"github.com/redis/go-redis/v9"
+	goredis "github.com/redis/go-redis/v9"
-	"go.unistack.org/micro/v4/options"
+	"go.unistack.org/micro/v3/logger"
+	"go.unistack.org/micro/v3/meter"
+	"go.unistack.org/micro/v3/store"
+	"go.unistack.org/micro/v3/tracer"
 )
 
+type configKey struct{}
+
+func Config(c *goredis.Options) store.Option {
+	return store.SetOption(configKey{}, c)
+}
+
+type clusterConfigKey struct{}
+
+func ClusterConfig(c *goredis.ClusterOptions) store.Option {
+	return store.SetOption(clusterConfigKey{}, c)
+}
+
 type universalConfigKey struct{}
 
-func UniversalConfig(c *redis.UniversalOptions) options.Option {
+func UniversalConfig(c *goredis.UniversalOptions) store.Option {
-	return options.ContextOption(universalConfigKey{}, c)
+	return store.SetOption(universalConfigKey{}, c)
+}
+
+var (
+	labelHost = "redis_host"
+	labelName = "redis_name"
+)
+
+// Options struct holds wrapper options
+type Options struct {
+	Logger    logger.Logger
+	Meter     meter.Meter
+	Tracer    tracer.Tracer
+	RedisHost string
+	RedisName string
+}
+
+// Option func signature
+type Option func(*Options)
+
+// NewOptions create new Options struct from provided option slice
+func NewOptions(opts ...Option) Options {
+	options := Options{
+		Logger: logger.DefaultLogger,
+		Meter:  meter.DefaultMeter,
+		Tracer: tracer.DefaultTracer,
+	}
+
+	for _, o := range opts {
+		o(&options)
+	}
+
+	options.Meter = options.Meter.Clone(
+		meter.Labels(
+			labelHost, options.RedisHost,
+			labelName, options.RedisName),
+	)
+
+	options.Logger = options.Logger.Clone(logger.WithAddCallerSkipCount(1))
+
+	return options
 }

redis_test.go

@@ -4,18 +4,67 @@ import (
 	"bytes"
 	"context"
 	"os"
+	"sync/atomic"
 	"testing"
 	"time"
 
-	"github.com/redis/go-redis/v9"
+	goredis "github.com/redis/go-redis/v9"
-	"go.unistack.org/micro/v4/options"
+	"go.unistack.org/micro/v3/store"
-	"go.unistack.org/micro/v4/store"
+	"go.unistack.org/micro/v3/tracer"
 )
 
+func TestLazyConnect(t *testing.T) {
+	t.Skip("skipping test for manual check")
+	ctx := context.Background()
+	var err error
+
+	r := NewStore()
+
+	if err = r.Init(); err != nil {
+		t.Fatal(err)
+	}
+	if err = r.Connect(ctx); err != nil {
+		t.Logf("connect failed %v", err)
+	}
+
+	for {
+		if err = r.Write(ctx, "mykey", "myval"); err != nil {
+			t.Logf("failed to write %v", err)
+		}
+	}
+}
+
+func TestKeepTTL(t *testing.T) {
+	ctx := context.Background()
+
+	if tr := os.Getenv("INTEGRATION_TESTS"); len(tr) > 0 {
+		t.Skip()
+	}
+	r := NewStore(store.Addrs(os.Getenv("STORE_NODES")))
+
+	if err := r.Init(store.LazyConnect(true)); err != nil {
+		t.Fatal(err)
+	}
+	if err := r.Connect(ctx); err != nil {
+		t.Fatal(err)
+	}
+
+	key := "key"
+	err := r.Write(ctx, key, "val1", store.WriteTTL(15*time.Second))
+	if err != nil {
+		t.Fatalf("Write error: %v", err)
+	}
+	time.Sleep(3 * time.Second)
+	err = r.Write(ctx, key, "val2", store.WriteTTL(-1))
+	if err != nil {
+		t.Fatalf("Write error: %v", err)
+	}
+}
+
 func Test_rkv_configure(t *testing.T) {
 	type fields struct {
 		options store.Options
-		Client  *redis.Client
+		Client  goredis.UniversalClient
 	}
 	type wantValues struct {
 		username string
@@ -38,7 +87,7 @@ func Test_rkv_configure(t *testing.T) {
 			},
 		},
 		{
-			name: "legacy Url", fields: fields{options: store.Options{Address: []string{"127.0.0.1:6379"}}, Client: nil},
+			name: "legacy Url", fields: fields{options: store.Options{Tracer: tracer.DefaultTracer, Addrs: []string{"127.0.0.1:6379"}}, Client: nil},
 			wantErr: false, want: wantValues{
 				username: "",
 				password: "",
@@ -46,7 +95,7 @@ func Test_rkv_configure(t *testing.T) {
 			},
 		},
 		{
-			name: "New Url", fields: fields{options: store.Options{Address: []string{"redis://127.0.0.1:6379"}}, Client: nil},
+			name: "New Url", fields: fields{options: store.Options{Tracer: tracer.DefaultTracer, Addrs: []string{"redis://127.0.0.1:6379"}}, Client: nil},
 			wantErr: false, want: wantValues{
 				username: "",
 				password: "",
@@ -54,7 +103,7 @@ func Test_rkv_configure(t *testing.T) {
 			},
 		},
 		{
-			name: "Url with Pwd", fields: fields{options: store.Options{Address: []string{"redis://:password@redis:6379"}}, Client: nil},
+			name: "Url with Pwd", fields: fields{options: store.Options{Tracer: tracer.DefaultTracer, Addrs: []string{"redis://:password@redis:6379"}}, Client: nil},
 			wantErr: false, want: wantValues{
 				username: "",
 				password: "password",
@@ -62,7 +111,7 @@ func Test_rkv_configure(t *testing.T) {
 			},
 		},
 		{
-			name: "Url with username and Pwd", fields: fields{options: store.Options{Address: []string{"redis://username:password@redis:6379"}}, Client: nil},
+			name: "Url with username and Pwd", fields: fields{options: store.Options{Tracer: tracer.DefaultTracer, Addrs: []string{"redis://username:password@redis:6379"}}, Client: nil},
 			wantErr: false, want: wantValues{
 				username: "username",
 				password: "password",
@@ -72,9 +121,11 @@ func Test_rkv_configure(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
+			b := atomic.Uint32{}
 			rc := &Store{
-				opts: tt.fields.options,
+				opts:      tt.fields.options,
-				cli:  tt.fields.Client,
+				cli:       tt.fields.Client,
+				connected: &b,
 			}
 			err := rc.configure()
 			if (err != nil) != tt.wantErr {
@@ -91,7 +142,7 @@ func Test_Store(t *testing.T) {
 	if tr := os.Getenv("INTEGRATION_TESTS"); len(tr) > 0 {
 		t.Skip()
 	}
-	r := NewStore(options.Address(os.Getenv("STORE_NODES")))
+	r := NewStore(store.Addrs(os.Getenv("STORE_NODES")))
 
 	if err := r.Init(); err != nil {
 		t.Fatal(err)
@@ -131,7 +182,7 @@ func Test_MRead(t *testing.T) {
 	if tr := os.Getenv("INTEGRATION_TESTS"); len(tr) > 0 {
 		t.Skip()
 	}
-	r := NewStore(options.Address(os.Getenv("STORE_NODES")))
+	r := NewStore(store.Addrs(os.Getenv("STORE_NODES")))
 
 	if err = r.Init(); err != nil {
 		t.Fatal(err)

stats.go

@@ -0,0 +1,49 @@
+package redis
+
+import (
+	"time"
+
+	goredis "github.com/redis/go-redis/v9"
+	"go.unistack.org/micro/v3/meter"
+)
+
+var (
+	PoolHitsTotal        = "pool_hits_total"
+	PoolMissesTotal      = "pool_misses_total"
+	PoolTimeoutTotal     = "pool_timeout_total"
+	PoolConnTotalCurrent = "pool_conn_total_current"
+	PoolConnIdleCurrent  = "pool_conn_idle_current"
+	PoolConnStaleTotal   = "pool_conn_stale_total"
+)
+
+type Statser interface {
+	PoolStats() *goredis.PoolStats
+}
+
+func (r *Store) statsMeter() {
+	var st Statser
+
+	if r.cli != nil {
+		st = r.cli
+	} else {
+		return
+	}
+
+	go func() {
+		ticker := time.NewTicker(meter.DefaultMeterStatsInterval)
+		defer ticker.Stop()
+
+		for _ = range ticker.C {
+			if st == nil {
+				return
+			}
+			stats := st.PoolStats()
+			r.opts.Meter.Counter(PoolHitsTotal).Set(uint64(stats.Hits))
+			r.opts.Meter.Counter(PoolMissesTotal).Set(uint64(stats.Misses))
+			r.opts.Meter.Counter(PoolTimeoutTotal).Set(uint64(stats.Timeouts))
+			r.opts.Meter.Counter(PoolConnTotalCurrent).Set(uint64(stats.TotalConns))
+			r.opts.Meter.Counter(PoolConnIdleCurrent).Set(uint64(stats.IdleConns))
+			r.opts.Meter.Counter(PoolConnStaleTotal).Set(uint64(stats.StaleConns))
+		}
+	}()
+}

tracer.go

@@ -0,0 +1,128 @@
+package redis
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"strconv"
+
+	rediscmd "github.com/redis/go-redis/extra/rediscmd/v9"
+	goredis "github.com/redis/go-redis/v9"
+	"go.unistack.org/micro/v3/tracer"
+)
+
+func setTracing(rdb goredis.UniversalClient, tr tracer.Tracer, opts ...tracer.SpanOption) {
+	switch rdb := rdb.(type) {
+	case *goredis.Client:
+		opt := rdb.Options()
+		connString := formatDBConnString(opt.Network, opt.Addr)
+		rdb.AddHook(newTracingHook(connString, tr))
+	case *goredis.ClusterClient:
+		rdb.OnNewNode(func(rdb *goredis.Client) {
+			opt := rdb.Options()
+			connString := formatDBConnString(opt.Network, opt.Addr)
+			rdb.AddHook(newTracingHook(connString, tr))
+		})
+	case *goredis.Ring:
+		rdb.OnNewNode(func(rdb *goredis.Client) {
+			opt := rdb.Options()
+			connString := formatDBConnString(opt.Network, opt.Addr)
+			rdb.AddHook(newTracingHook(connString, tr))
+		})
+	}
+}
+
+type tracingHook struct {
+	tr   tracer.Tracer
+	opts []tracer.SpanOption
+}
+
+var _ goredis.Hook = (*tracingHook)(nil)
+
+func newTracingHook(connString string, tr tracer.Tracer, opts ...tracer.SpanOption) *tracingHook {
+	opts = append(opts, tracer.WithSpanKind(tracer.SpanKindClient))
+	if connString != "" {
+		opts = append(opts, tracer.WithSpanLabels("db.connection_string", connString))
+	}
+
+	return &tracingHook{
+		tr:   tr,
+		opts: opts,
+	}
+}
+
+func (h *tracingHook) DialHook(hook goredis.DialHook) goredis.DialHook {
+	return func(ctx context.Context, network, addr string) (net.Conn, error) {
+		/*
+			_, span := h.tr.Start(ctx, "goredis.dial", h.opts...)
+			defer span.Finish()
+		*/
+		conn, err := hook(ctx, network, addr)
+		// recordError(span, err)
+
+		return conn, err
+	}
+}
+
+func (h *tracingHook) ProcessHook(hook goredis.ProcessHook) goredis.ProcessHook {
+	return func(ctx context.Context, cmd goredis.Cmder) error {
+		cmdString := rediscmd.CmdString(cmd)
+		var err error
+
+		switch cmdString {
+		case "cluster slots":
+			break
+		default:
+			_, span := h.tr.Start(ctx, "sdk.database", append(h.opts, tracer.WithSpanLabels("db.statement", cmdString))...)
+			defer func() {
+				recordError(span, err)
+				span.Finish()
+			}()
+		}
+
+		err = hook(ctx, cmd)
+
+		return err
+	}
+}
+
+func (h *tracingHook) ProcessPipelineHook(hook goredis.ProcessPipelineHook) goredis.ProcessPipelineHook {
+	return func(ctx context.Context, cmds []goredis.Cmder) error {
+		_, cmdsString := rediscmd.CmdsString(cmds)
+
+		opts := append(h.opts, tracer.WithSpanLabels(
+			"db.database.num_cmd", strconv.Itoa(len(cmds)),
+			"db.statement", cmdsString,
+		))
+
+		_, span := h.tr.Start(ctx, "sdk.database", opts...)
+		defer span.Finish()
+
+		err := hook(ctx, cmds)
+		recordError(span, err)
+
+		return err
+	}
+}
+
+func setSpanError(ctx context.Context, err error) {
+	if err == nil || err == goredis.Nil {
+		return
+	}
+	if sp, ok := tracer.SpanFromContext(ctx); !ok && sp != nil {
+		sp.SetStatus(tracer.SpanStatusError, err.Error())
+	}
+}
+
+func recordError(span tracer.Span, err error) {
+	if err != nil && err != goredis.Nil {
+		span.SetStatus(tracer.SpanStatusError, err.Error())
+	}
+}
+
+func formatDBConnString(network, addr string) string {
+	if network == "tcp" {
+		network = "redis"
+	}
+	return fmt.Sprintf("%s://%s", network, addr)
+}