add store.Watcher interface (#123)

Signed-off-by: Vasiliy Tolstov <v.tolstov@unistack.org>
Reviewed-on: #123
Co-authored-by: Vasiliy Tolstov <v.tolstov@unistack.org>
Co-committed-by: Vasiliy Tolstov <v.tolstov@unistack.org>

event.go

@@ -0,0 +1,86 @@
+package redis
+
+import (
+	"context"
+	"errors"
+	"net"
+	"time"
+
+	goredis "github.com/redis/go-redis/v9"
+	"go.unistack.org/micro/v3/store"
+)
+
+type eventHook struct {
+	s *Store
+}
+
+var _ goredis.Hook = (*eventHook)(nil)
+
+func newEventHook(s *Store) *eventHook {
+	return &eventHook{s: s}
+}
+
+func (h *eventHook) DialHook(hook goredis.DialHook) goredis.DialHook {
+	return func(ctx context.Context, network, addr string) (net.Conn, error) {
+		conn, err := hook(ctx, network, addr)
+		if err != nil {
+			if !isRedisError(err) {
+				if h.s.connected.CompareAndSwap(1, 0) {
+					h.s.sendEvent(&event{ts: time.Now(), err: err, t: store.EventTypeDisconnect})
+				}
+			} else {
+				h.s.connected.Store(1)
+			}
+		} else {
+			if h.s.connected.CompareAndSwap(0, 1) {
+				h.s.sendEvent(&event{ts: time.Now(), err: err, t: store.EventTypeConnect})
+			}
+		}
+		return conn, err
+	}
+}
+
+func (h *eventHook) ProcessHook(hook goredis.ProcessHook) goredis.ProcessHook {
+	return func(ctx context.Context, cmd goredis.Cmder) error {
+		err := hook(ctx, cmd)
+		if err != nil {
+			if !isRedisError(err) {
+				if h.s.connected.CompareAndSwap(1, 0) {
+					h.s.sendEvent(&event{ts: time.Now(), err: err, t: store.EventTypeDisconnect})
+				}
+			} else {
+				h.s.connected.Store(1)
+			}
+		} else {
+			if h.s.connected.CompareAndSwap(0, 1) {
+				h.s.sendEvent(&event{ts: time.Now(), err: err, t: store.EventTypeConnect})
+			}
+		}
+		return err
+	}
+}
+
+func (h *eventHook) ProcessPipelineHook(hook goredis.ProcessPipelineHook) goredis.ProcessPipelineHook {
+	return func(ctx context.Context, cmds []goredis.Cmder) error {
+		err := hook(ctx, cmds)
+		if err != nil {
+			if !isRedisError(err) {
+				if h.s.connected.CompareAndSwap(1, 0) {
+					h.s.sendEvent(&event{ts: time.Now(), err: err, t: store.EventTypeDisconnect})
+				}
+			} else {
+				h.s.connected.Store(1)
+			}
+		} else {
+			if h.s.connected.CompareAndSwap(0, 1) {
+				h.s.sendEvent(&event{ts: time.Now(), err: err, t: store.EventTypeConnect})
+			}
+		}
+		return err
+	}
+}
+
+func isRedisError(err error) bool {
+	var rerr goredis.Error
+	return errors.As(err, &rerr)
+}

go.mod

@@ -5,9 +5,9 @@ go 1.22
 toolchain go1.22.4
 
 require (
-	github.com/redis/go-redis/extra/rediscmd/v9 v9.0.5
-	github.com/redis/go-redis/v9 v9.6.1
-	go.unistack.org/micro/v3 v3.10.96
+	github.com/redis/go-redis/extra/rediscmd/v9 v9.7.0
+	github.com/redis/go-redis/v9 v9.7.0
+	go.unistack.org/micro/v3 v3.10.108
 )
 
 require (
@@ -15,5 +15,5 @@ require (
 	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
 	github.com/google/go-cmp v0.6.0 // indirect
 	go.unistack.org/micro-proto/v3 v3.4.1 // indirect
-	google.golang.org/protobuf v1.35.1 // indirect
+	google.golang.org/protobuf v1.35.2 // indirect
 )

go.sum

@@ -1,24 +1,20 @@
-github.com/bsm/ginkgo/v2 v2.7.0/go.mod h1:AiKlXPm7ItEHNc/2+OkrNG4E0ITzojb9/xWzvQ9XZ9w=
 github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
 github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=
-github.com/bsm/gomega v1.26.0/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0=
 github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
 github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0=
-github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
 github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
 github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
 github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
 github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
-github.com/redis/go-redis/extra/rediscmd/v9 v9.0.5 h1:EaDatTxkdHG+U3Bk4EUr+DZ7fOGwTfezUiUJMaIcaho=
-github.com/redis/go-redis/extra/rediscmd/v9 v9.0.5/go.mod h1:fyalQWdtzDBECAQFBJuQe5bzQ02jGd5Qcbgb97Flm7U=
-github.com/redis/go-redis/v9 v9.0.5/go.mod h1:WqMKv5vnQbRuZstUwxQI195wHy+t4PuXDOjzMvcuQHk=
-github.com/redis/go-redis/v9 v9.6.1 h1:HHDteefn6ZkTtY5fGUE8tj8uy85AHk6zP7CpzIAM0y4=
-github.com/redis/go-redis/v9 v9.6.1/go.mod h1:0C0c6ycQsdpVNQpxb1njEQIqkx5UcsM8FJCQLgE9+RA=
+github.com/redis/go-redis/extra/rediscmd/v9 v9.7.0 h1:BIx9TNZH/Jsr4l1i7VVxnV0JPiwYj8qyrHyuL0fGZrk=
+github.com/redis/go-redis/extra/rediscmd/v9 v9.7.0/go.mod h1:eTg/YQtGYAZD5r3DlGlJptJ45AHA+/G+2NPn30PKzik=
+github.com/redis/go-redis/v9 v9.7.0 h1:HhLSs+B6O021gwzl+locl0zEDnyNkxMtf/Z3NNBMa9E=
+github.com/redis/go-redis/v9 v9.7.0/go.mod h1:f6zhXITC7JUJIlPEiBOTXxJgPLdZcA93GewI7inzyWw=
 go.unistack.org/micro-proto/v3 v3.4.1 h1:UTjLSRz2YZuaHk9iSlVqqsA50JQNAEK2ZFboGqtEa9Q=
 go.unistack.org/micro-proto/v3 v3.4.1/go.mod h1:okx/cnOhzuCX0ggl/vToatbCupi0O44diiiLLsZ93Zo=
-go.unistack.org/micro/v3 v3.10.96 h1:njukb0DlXOJspX5mLSwcP7+cVSg/8Fy+6++mYkOpY2A=
-go.unistack.org/micro/v3 v3.10.96/go.mod h1:YzMldzHN9Ei+zy5t/Psu7RUWDZwUfrNYiStSQtTz90g=
-google.golang.org/protobuf v1.35.1 h1:m3LfL6/Ca+fqnjnlqQXNpFPABW1UD7mjh8KO2mKFytA=
-google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE=
+go.unistack.org/micro/v3 v3.10.108 h1:3L7SkilMVLtH8y3pQIPtr3jjQYrf0AMv1oAkoL3nFkE=
+go.unistack.org/micro/v3 v3.10.108/go.mod h1:YzMldzHN9Ei+zy5t/Psu7RUWDZwUfrNYiStSQtTz90g=
+google.golang.org/protobuf v1.35.2 h1:8Ar7bF+apOIoThw1EdZl0p1oWvMqTHmpA2fRTyZO8io=
+google.golang.org/protobuf v1.35.2/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE=

options.go

@@ -61,7 +61,7 @@ func NewOptions(opts ...Option) Options {
 			labelName, options.RedisName),
 	)
 
-	options.Logger = options.Logger.Clone(logger.WithCallerSkipCount(1))
+	options.Logger = options.Logger.Clone(logger.WithAddCallerSkipCount(1))
 
 	return options
 }

redis.go

@@ -2,20 +2,25 @@ package redis
 
 import (
 	"context"
+	"errors"
 	"reflect"
 	"strings"
+	"sync"
+	"sync/atomic"
 	"time"
 
 	goredis "github.com/redis/go-redis/v9"
 	"go.unistack.org/micro/v3/semconv"
 	"go.unistack.org/micro/v3/store"
+	"go.unistack.org/micro/v3/util/id"
 	pool "go.unistack.org/micro/v3/util/xpool"
 )
 
 var (
-	DefaultPathSeparator = "/"
-
-	DefaultUniversalOptions = &goredis.UniversalOptions{
+	_                       store.Store = (*Store)(nil)
+	_                       store.Event = (*event)(nil)
+	sendEventTime                       = 10 * time.Millisecond
+	DefaultUniversalOptions             = &goredis.UniversalOptions{
 		Username:        "",
 		Password:        "", // no password set
 		MaxRetries:      2,
@@ -54,19 +59,30 @@ var (
 )
 
 type Store struct {
-	opts store.Options
-	cli  goredis.UniversalClient
-	done chan struct{}
-	pool *pool.StringsPool
+	cli       goredis.UniversalClient
+	pool      *pool.StringsPool
+	connected *atomic.Uint32
+	opts      store.Options
+	watchers  map[string]*watcher
+	mu        sync.RWMutex
 }
 
 func (r *Store) Connect(ctx context.Context) error {
+	if r.connected.Load() == 1 {
+		return nil
+	}
 	if r.cli == nil {
 		return store.ErrNotConnected
 	}
-	err := r.cli.Ping(ctx).Err()
-	setSpanError(ctx, err)
-	return err
+	if r.opts.LazyConnect {
+		return nil
+	}
+	if err := r.cli.Ping(ctx).Err(); err != nil {
+		setSpanError(ctx, err)
+		return err
+	}
+	r.connected.Store(1)
+	return nil
 }
 
 func (r *Store) Init(opts ...store.Option) error {
@@ -101,23 +117,26 @@ func (r *Store) ClusterClient() *goredis.ClusterClient {
 }
 
 func (r *Store) Disconnect(ctx context.Context) error {
-	var err error
-	select {
-	case <-r.done:
-		return err
-	default:
-		if r.cli != nil {
-			err = r.cli.Close()
-		}
-		close(r.done)
-		return err
+	if r.connected.Load() == 0 {
+		return nil
 	}
+
+	if r.cli != nil {
+		if err := r.cli.Close(); err != nil {
+			return err
+		}
+	}
+
+	r.connected.Store(1)
+	return nil
 }
 
 func (r *Store) Exists(ctx context.Context, key string, opts ...store.ExistsOption) error {
 	b := r.pool.Get()
 	defer r.pool.Put(b)
 	options := store.NewExistsOptions(opts...)
+	labels := make([]string, 0, 6)
+	labels = append(labels, "name", options.Name, "statement", "exists")
 
 	timeout := r.opts.Timeout
 	if options.Timeout > 0 {
@@ -132,21 +151,22 @@ func (r *Store) Exists(ctx context.Context, key string, opts ...store.ExistsOpti
 
 	rkey := r.getKey(b, r.opts.Namespace, options.Namespace, key)
 
-	r.opts.Meter.Counter(semconv.StoreRequestInflight, "name", options.Name).Inc()
+	r.opts.Meter.Counter(semconv.StoreRequestInflight, labels...).Inc()
 	ts := time.Now()
 	val, err := r.cli.Exists(ctx, rkey).Result()
 	setSpanError(ctx, err)
 	te := time.Since(ts)
-	r.opts.Meter.Counter(semconv.StoreRequestInflight, "name", options.Name).Dec()
-	r.opts.Meter.Summary(semconv.StoreRequestLatencyMicroseconds, "name", options.Name).Update(te.Seconds())
-	r.opts.Meter.Histogram(semconv.StoreRequestDurationSeconds, "name", options.Name).Update(te.Seconds())
-	if err == goredis.Nil || (err == nil && val == 0) {
-		r.opts.Meter.Counter(semconv.StoreRequestTotal, "name", options.Name, "status", "miss").Inc()
+	r.opts.Meter.Counter(semconv.StoreRequestInflight, labels...).Dec()
+
+	r.opts.Meter.Summary(semconv.StoreRequestLatencyMicroseconds, labels...).Update(te.Seconds())
+	r.opts.Meter.Histogram(semconv.StoreRequestDurationSeconds, labels...).Update(te.Seconds())
+	if errors.Is(err, goredis.Nil) || (err == nil && val == 0) {
+		r.opts.Meter.Counter(semconv.StoreRequestTotal, append(labels, "status", "miss")...).Inc()
 		return store.ErrNotFound
 	} else if err == nil {
-		r.opts.Meter.Counter(semconv.StoreRequestTotal, "name", options.Name, "status", "hit").Inc()
-	} else if err != nil {
-		r.opts.Meter.Counter(semconv.StoreRequestTotal, "name", options.Name, "status", "failure").Inc()
+		r.opts.Meter.Counter(semconv.StoreRequestTotal, append(labels, "status", "hit")...).Inc()
+	} else {
+		r.opts.Meter.Counter(semconv.StoreRequestTotal, append(labels, "status", "failure")...).Inc()
 		return err
 	}
 
@@ -158,6 +178,8 @@ func (r *Store) Read(ctx context.Context, key string, val interface{}, opts ...s
 	defer r.pool.Put(b)
 
 	options := store.NewReadOptions(opts...)
+	labels := make([]string, 0, 6)
+	labels = append(labels, "name", options.Name, "statement", "read")
 
 	timeout := r.opts.Timeout
 	if options.Timeout > 0 {
@@ -172,21 +194,21 @@ func (r *Store) Read(ctx context.Context, key string, val interface{}, opts ...s
 
 	rkey := r.getKey(b, r.opts.Namespace, options.Namespace, key)
 
-	r.opts.Meter.Counter(semconv.StoreRequestInflight, "name", options.Name).Inc()
+	r.opts.Meter.Counter(semconv.StoreRequestInflight, labels...).Inc()
 	ts := time.Now()
 	buf, err := r.cli.Get(ctx, rkey).Bytes()
 	setSpanError(ctx, err)
 	te := time.Since(ts)
-	r.opts.Meter.Counter(semconv.StoreRequestInflight, "name", options.Name).Dec()
-	r.opts.Meter.Summary(semconv.StoreRequestLatencyMicroseconds, "name", options.Name).Update(te.Seconds())
-	r.opts.Meter.Histogram(semconv.StoreRequestDurationSeconds, "name", options.Name).Update(te.Seconds())
-	if err == goredis.Nil || (err == nil && buf == nil) {
-		r.opts.Meter.Counter(semconv.StoreRequestTotal, "name", options.Name, "status", "miss").Inc()
+	r.opts.Meter.Counter(semconv.StoreRequestInflight, labels...).Dec()
+	r.opts.Meter.Summary(semconv.StoreRequestLatencyMicroseconds, labels...).Update(te.Seconds())
+	r.opts.Meter.Histogram(semconv.StoreRequestDurationSeconds, labels...).Update(te.Seconds())
+	if errors.Is(err, goredis.Nil) || (err == nil && buf == nil) {
+		r.opts.Meter.Counter(semconv.StoreRequestTotal, append(labels, "status", "miss")...).Inc()
 		return store.ErrNotFound
 	} else if err == nil {
-		r.opts.Meter.Counter(semconv.StoreRequestTotal, "name", options.Name, "status", "hit").Inc()
+		r.opts.Meter.Counter(semconv.StoreRequestTotal, append(labels, "status", "hit")...).Inc()
 	} else if err != nil {
-		r.opts.Meter.Counter(semconv.StoreRequestTotal, "name", options.Name, "status", "failure").Inc()
+		r.opts.Meter.Counter(semconv.StoreRequestTotal, append(labels, "status", "failure")...).Inc()
 		return err
 	}
 
@@ -305,6 +327,8 @@ func (r *Store) MRead(ctx context.Context, keys []string, vals interface{}, opts
 
 func (r *Store) MDelete(ctx context.Context, keys []string, opts ...store.DeleteOption) error {
 	options := store.NewDeleteOptions(opts...)
+	labels := make([]string, 0, 6)
+	labels = append(labels, "name", options.Name, "statement", "delete")
 
 	timeout := r.opts.Timeout
 	if options.Timeout > 0 {
@@ -329,7 +353,7 @@ func (r *Store) MDelete(ctx context.Context, keys []string, opts ...store.Delete
 		}
 	}
 
-	r.opts.Meter.Counter(semconv.StoreRequestInflight, "name", options.Name).Inc()
+	r.opts.Meter.Counter(semconv.StoreRequestInflight, labels...).Inc()
 	ts := time.Now()
 	var err error
 	if r.opts.Namespace != "" || options.Namespace != "" {
@@ -342,16 +366,16 @@ func (r *Store) MDelete(ctx context.Context, keys []string, opts ...store.Delete
 	}
 	setSpanError(ctx, err)
 	te := time.Since(ts)
-	r.opts.Meter.Counter(semconv.StoreRequestInflight, "name", options.Name).Dec()
-	r.opts.Meter.Summary(semconv.StoreRequestLatencyMicroseconds, "name", options.Name).Update(te.Seconds())
-	r.opts.Meter.Histogram(semconv.StoreRequestDurationSeconds, "name", options.Name).Update(te.Seconds())
+	r.opts.Meter.Counter(semconv.StoreRequestInflight, labels...).Dec()
+	r.opts.Meter.Summary(semconv.StoreRequestLatencyMicroseconds, labels...).Update(te.Seconds())
+	r.opts.Meter.Histogram(semconv.StoreRequestDurationSeconds, labels...).Update(te.Seconds())
 	if err == goredis.Nil {
-		r.opts.Meter.Counter(semconv.StoreRequestTotal, "name", options.Name, "status", "miss").Inc()
+		r.opts.Meter.Counter(semconv.StoreRequestTotal, append(labels, "status", "miss")...).Inc()
 		return store.ErrNotFound
 	} else if err == nil {
-		r.opts.Meter.Counter(semconv.StoreRequestTotal, "name", options.Name, "status", "hit").Inc()
+		r.opts.Meter.Counter(semconv.StoreRequestTotal, append(labels, "status", "hit")...).Inc()
 	} else if err != nil {
-		r.opts.Meter.Counter(semconv.StoreRequestTotal, "name", options.Name, "status", "failure").Inc()
+		r.opts.Meter.Counter(semconv.StoreRequestTotal, append(labels, "status", "failure")...).Inc()
 		return err
 	}
 
@@ -363,6 +387,8 @@ func (r *Store) Delete(ctx context.Context, key string, opts ...store.DeleteOpti
 	defer r.pool.Put(b)
 
 	options := store.NewDeleteOptions(opts...)
+	labels := make([]string, 0, 6)
+	labels = append(labels, "name", options.Name, "statement", "delete")
 
 	timeout := r.opts.Timeout
 	if options.Timeout > 0 {
@@ -375,21 +401,22 @@ func (r *Store) Delete(ctx context.Context, key string, opts ...store.DeleteOpti
 		defer cancel()
 	}
 
-	r.opts.Meter.Counter(semconv.StoreRequestInflight, "name", options.Name).Inc()
+	r.opts.Meter.Counter(semconv.StoreRequestInflight, labels...).Inc()
 	ts := time.Now()
 	err := r.cli.Del(ctx, r.getKey(b, r.opts.Namespace, options.Namespace, key)).Err()
 	te := time.Since(ts)
 	setSpanError(ctx, err)
-	r.opts.Meter.Counter(semconv.StoreRequestInflight, "name", options.Name).Dec()
-	r.opts.Meter.Summary(semconv.StoreRequestLatencyMicroseconds, "name", options.Name).Update(te.Seconds())
-	r.opts.Meter.Histogram(semconv.StoreRequestDurationSeconds, "name", options.Name).Update(te.Seconds())
-	if err == goredis.Nil {
-		r.opts.Meter.Counter(semconv.StoreRequestTotal, "name", options.Name, "status", "miss").Inc()
+
+	r.opts.Meter.Counter(semconv.StoreRequestInflight, labels...).Dec()
+	r.opts.Meter.Summary(semconv.StoreRequestLatencyMicroseconds, labels...).Update(te.Seconds())
+	r.opts.Meter.Histogram(semconv.StoreRequestDurationSeconds, labels...).Update(te.Seconds())
+	if errors.Is(err, goredis.Nil) {
+		r.opts.Meter.Counter(semconv.StoreRequestTotal, append(labels, "status", "miss")...).Inc()
 		return store.ErrNotFound
 	} else if err == nil {
-		r.opts.Meter.Counter(semconv.StoreRequestTotal, "name", options.Name, "status", "hit").Inc()
+		r.opts.Meter.Counter(semconv.StoreRequestTotal, append(labels, "status", "hit")...).Inc()
 	} else if err != nil {
-		r.opts.Meter.Counter(semconv.StoreRequestTotal, "name", options.Name, "status", "failure").Inc()
+		r.opts.Meter.Counter(semconv.StoreRequestTotal, append(labels, "status", "failure")...).Inc()
 		return err
 	}
 
@@ -398,6 +425,8 @@ func (r *Store) Delete(ctx context.Context, key string, opts ...store.DeleteOpti
 
 func (r *Store) MWrite(ctx context.Context, keys []string, vals []interface{}, opts ...store.WriteOption) error {
 	options := store.NewWriteOptions(opts...)
+	labels := make([]string, 0, 6)
+	labels = append(labels, "name", options.Name, "statement", "write")
 
 	timeout := r.opts.Timeout
 	if options.Timeout > 0 {
@@ -431,7 +460,7 @@ func (r *Store) MWrite(ctx context.Context, keys []string, vals []interface{}, o
 		}
 	}
 
-	r.opts.Meter.Counter(semconv.StoreRequestInflight, "name", options.Name).Inc()
+	r.opts.Meter.Counter(semconv.StoreRequestInflight, labels...).Inc()
 
 	pipeliner := func(pipe goredis.Pipeliner) error {
 		for idx := 0; idx < len(kvs); idx += 2 {
@@ -451,27 +480,27 @@ func (r *Store) MWrite(ctx context.Context, keys []string, vals []interface{}, o
 	te := time.Since(ts)
 	setSpanError(ctx, err)
 
-	r.opts.Meter.Counter(semconv.StoreRequestInflight, "name", options.Name).Dec()
-	r.opts.Meter.Summary(semconv.StoreRequestLatencyMicroseconds, "name", options.Name).Update(te.Seconds())
-	r.opts.Meter.Histogram(semconv.StoreRequestDurationSeconds, "name", options.Name).Update(te.Seconds())
+	r.opts.Meter.Counter(semconv.StoreRequestInflight, labels...).Dec()
+	r.opts.Meter.Summary(semconv.StoreRequestLatencyMicroseconds, labels...).Update(te.Seconds())
+	r.opts.Meter.Histogram(semconv.StoreRequestDurationSeconds, labels...).Update(te.Seconds())
 	if err == goredis.Nil {
-		r.opts.Meter.Counter(semconv.StoreRequestTotal, "name", options.Name, "status", "miss").Inc()
+		r.opts.Meter.Counter(semconv.StoreRequestTotal, append(labels, "status", "miss")...).Inc()
 		return store.ErrNotFound
 	} else if err == nil {
-		r.opts.Meter.Counter(semconv.StoreRequestTotal, "name", options.Name, "status", "hit").Inc()
+		r.opts.Meter.Counter(semconv.StoreRequestTotal, append(labels, "status", "hit")...).Inc()
 	} else if err != nil {
-		r.opts.Meter.Counter(semconv.StoreRequestTotal, "name", options.Name, "status", "failure").Inc()
+		r.opts.Meter.Counter(semconv.StoreRequestTotal, append(labels, "status", "failure")...).Inc()
 		return err
 	}
 
 	for _, cmd := range cmds {
 		if err = cmd.Err(); err != nil {
 			if err == goredis.Nil {
-				r.opts.Meter.Counter(semconv.StoreRequestTotal, "name", options.Name, "status", "miss").Inc()
+				r.opts.Meter.Counter(semconv.StoreRequestTotal, append(labels, "status", "miss")...).Inc()
 				return store.ErrNotFound
 			}
 			setSpanError(ctx, err)
-			r.opts.Meter.Counter(semconv.StoreRequestTotal, "name", options.Name, "status", "failure").Inc()
+			r.opts.Meter.Counter(semconv.StoreRequestTotal, append(labels, "status", "failure")...).Inc()
 			return err
 		}
 	}
@@ -484,6 +513,8 @@ func (r *Store) Write(ctx context.Context, key string, val interface{}, opts ...
 	defer r.pool.Put(b)
 
 	options := store.NewWriteOptions(opts...)
+	labels := make([]string, 0, 6)
+	labels = append(labels, "name", options.Name, "statement", "write")
 
 	timeout := r.opts.Timeout
 	if options.Timeout > 0 {
@@ -512,22 +543,22 @@ func (r *Store) Write(ctx context.Context, key string, val interface{}, opts ...
 		}
 	}
 
-	r.opts.Meter.Counter(semconv.StoreRequestInflight, "name", options.Name).Inc()
+	r.opts.Meter.Counter(semconv.StoreRequestInflight, labels...).Inc()
 	ts := time.Now()
 	err := r.cli.Set(ctx, rkey, buf, options.TTL).Err()
 	te := time.Since(ts)
 	setSpanError(ctx, err)
 
-	r.opts.Meter.Counter(semconv.StoreRequestInflight, "name", options.Name).Dec()
-	r.opts.Meter.Summary(semconv.StoreRequestLatencyMicroseconds, "name", options.Name).Update(te.Seconds())
-	r.opts.Meter.Histogram(semconv.StoreRequestDurationSeconds, "name", options.Name).Update(te.Seconds())
-	if err == goredis.Nil {
-		r.opts.Meter.Counter(semconv.StoreRequestTotal, "name", options.Name, "status", "miss").Inc()
+	r.opts.Meter.Counter(semconv.StoreRequestInflight, labels...).Dec()
+	r.opts.Meter.Summary(semconv.StoreRequestLatencyMicroseconds, labels...).Update(te.Seconds())
+	r.opts.Meter.Histogram(semconv.StoreRequestDurationSeconds, labels...).Update(te.Seconds())
+	if errors.Is(err, goredis.Nil) {
+		r.opts.Meter.Counter(semconv.StoreRequestTotal, append(labels, "status", "miss")...).Inc()
 		return store.ErrNotFound
 	} else if err == nil {
-		r.opts.Meter.Counter(semconv.StoreRequestTotal, "name", options.Name, "status", "hit").Inc()
-	} else if err != nil {
-		r.opts.Meter.Counter(semconv.StoreRequestTotal, "name", options.Name, "status", "failure").Inc()
+		r.opts.Meter.Counter(semconv.StoreRequestTotal, append(labels, "status", "hit")...).Inc()
+	} else {
+		r.opts.Meter.Counter(semconv.StoreRequestTotal, append(labels, "status", "failure")...).Inc()
 		return err
 	}
 
@@ -539,6 +570,9 @@ func (r *Store) List(ctx context.Context, opts ...store.ListOption) ([]string, e
 	defer r.pool.Put(b)
 
 	options := store.NewListOptions(opts...)
+	labels := make([]string, 0, 6)
+	labels = append(labels, "name", options.Name, "statement", "list")
+
 	if len(options.Namespace) == 0 {
 		options.Namespace = r.opts.Namespace
 	}
@@ -560,7 +594,7 @@ func (r *Store) List(ctx context.Context, opts ...store.ListOption) ([]string, e
 	}
 
 	// TODO: add support for prefix/suffix/limit
-	r.opts.Meter.Counter(semconv.StoreRequestInflight, "name", options.Name).Inc()
+	r.opts.Meter.Counter(semconv.StoreRequestInflight, labels...).Inc()
 	ts := time.Now()
 	var keys []string
 	var err error
@@ -580,16 +614,16 @@ func (r *Store) List(ctx context.Context, opts ...store.ListOption) ([]string, e
 	te := time.Since(ts)
 	setSpanError(ctx, err)
 
-	r.opts.Meter.Counter(semconv.StoreRequestInflight, "name", options.Name).Dec()
-	r.opts.Meter.Summary(semconv.StoreRequestLatencyMicroseconds, "name", options.Name).Update(te.Seconds())
-	r.opts.Meter.Histogram(semconv.StoreRequestDurationSeconds, "name", options.Name).Update(te.Seconds())
+	r.opts.Meter.Counter(semconv.StoreRequestInflight, labels...).Dec()
+	r.opts.Meter.Summary(semconv.StoreRequestLatencyMicroseconds, labels...).Update(te.Seconds())
+	r.opts.Meter.Histogram(semconv.StoreRequestDurationSeconds, labels...).Update(te.Seconds())
 	if err == goredis.Nil {
-		r.opts.Meter.Counter(semconv.StoreRequestTotal, "name", options.Name, "status", "miss").Inc()
+		r.opts.Meter.Counter(semconv.StoreRequestTotal, append(labels, "status", "miss")...).Inc()
 		return nil, store.ErrNotFound
 	} else if err == nil {
-		r.opts.Meter.Counter(semconv.StoreRequestTotal, "name", options.Name, "status", "hit").Inc()
+		r.opts.Meter.Counter(semconv.StoreRequestTotal, append(labels, "status", "git")...).Inc()
 	} else if err != nil {
-		r.opts.Meter.Counter(semconv.StoreRequestTotal, "name", options.Name, "status", "failure").Inc()
+		r.opts.Meter.Counter(semconv.StoreRequestTotal, append(labels, "status", "failure")...).Inc()
 		return nil, err
 	}
 
@@ -621,16 +655,21 @@ func (r *Store) String() string {
 }
 
 func NewStore(opts ...store.Option) *Store {
-	return &Store{done: make(chan struct{}), opts: store.NewOptions(opts...)}
+	b := atomic.Uint32{}
+	return &Store{
+		opts:      store.NewOptions(opts...),
+		connected: &b,
+		watchers:  make(map[string]*watcher),
+	}
 }
 
 func (r *Store) configure() error {
-	var universalOptions *goredis.UniversalOptions
-
 	if r.cli != nil && r.opts.Context == nil {
 		return nil
 	}
 
+	universalOptions := DefaultUniversalOptions
+
 	if r.opts.Context != nil {
 		if o, ok := r.opts.Context.Value(configKey{}).(*goredis.Options); ok {
 			universalOptions.Addrs = []string{o.Addr}
@@ -704,10 +743,6 @@ func (r *Store) configure() error {
 		}
 	}
 
-	if universalOptions == nil {
-		universalOptions = DefaultUniversalOptions
-	}
-
 	if len(r.opts.Addrs) > 0 {
 		universalOptions.Addrs = r.opts.Addrs
 	} else {
@@ -716,6 +751,7 @@ func (r *Store) configure() error {
 
 	r.cli = goredis.NewUniversalClient(universalOptions)
 	setTracing(r.cli, r.opts.Tracer)
+	r.cli.AddHook(newEventHook(r))
 
 	r.pool = pool.NewStringsPool(50)
 
@@ -730,8 +766,99 @@ func (r *Store) getKey(b *strings.Builder, mainNamespace string, opNamespace str
 	}
 	if opNamespace != "" {
 		b.WriteString(opNamespace)
-		b.WriteString(DefaultPathSeparator)
+		b.WriteString(r.opts.Separator)
 	}
 	b.WriteString(key)
 	return b.String()
 }
+
+func (r *Store) Watch(ctx context.Context, opts ...store.WatchOption) (store.Watcher, error) {
+	id, err := id.New()
+	if err != nil {
+		return nil, err
+	}
+	wo, err := store.NewWatchOptions(opts...)
+	if err != nil {
+		return nil, err
+	}
+	// construct the watcher
+	w := &watcher{
+		exit: make(chan bool),
+		ch:   make(chan store.Event),
+		id:   id,
+		opts: wo,
+	}
+
+	r.mu.Lock()
+	r.watchers[w.id] = w
+	r.mu.Unlock()
+
+	return w, nil
+}
+
+func (r *Store) sendEvent(e store.Event) {
+	r.mu.RLock()
+	watchers := make([]*watcher, 0, len(r.watchers))
+	for _, w := range r.watchers {
+		watchers = append(watchers, w)
+	}
+	r.mu.RUnlock()
+	for _, w := range watchers {
+		select {
+		case <-w.exit:
+			r.mu.Lock()
+			delete(r.watchers, w.id)
+			r.mu.Unlock()
+		default:
+			select {
+			case w.ch <- e:
+			case <-time.After(sendEventTime):
+			}
+		}
+	}
+}
+
+type watcher struct {
+	ch   chan store.Event
+	exit chan bool
+	opts store.WatchOptions
+	id   string
+}
+
+func (w *watcher) Next() (store.Event, error) {
+	for {
+		select {
+		case e := <-w.ch:
+			return e, nil
+		case <-w.exit:
+			return nil, store.ErrWatcherStopped
+		}
+	}
+}
+
+func (w *watcher) Stop() {
+	select {
+	case <-w.exit:
+		return
+	default:
+		close(w.exit)
+	}
+}
+
+type event struct {
+	ts  time.Time
+	t   store.EventType
+	err error
+}
+
+func (e *event) Error() error {
+	return e.err
+}
+
+func (e *event) Timestamp() time.Time {
+	return e.ts
+}
+
+func (e *event) Type() store.EventType {
+	return e.t
+}

redis_test.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"context"
 	"os"
+	"sync/atomic"
 	"testing"
 	"time"
 
@@ -12,6 +13,26 @@ import (
 	"go.unistack.org/micro/v3/tracer"
 )
 
+func TestLazyConnect(t *testing.T) {
+	ctx := context.Background()
+	var err error
+
+	r := NewStore()
+
+	if err = r.Init(); err != nil {
+		t.Fatal(err)
+	}
+	if err = r.Connect(ctx); err != nil {
+		t.Logf("connect failed %v", err)
+	}
+
+	for {
+		if err = r.Write(ctx, "mykey", "myval"); err != nil {
+			t.Logf("failed to write %v", err)
+		}
+	}
+}
+
 func TestKeepTTL(t *testing.T) {
 	ctx := context.Background()
 
@@ -20,7 +41,7 @@ func TestKeepTTL(t *testing.T) {
 	}
 	r := NewStore(store.Addrs(os.Getenv("STORE_NODES")))
 
-	if err := r.Init(); err != nil {
+	if err := r.Init(store.LazyConnect(true)); err != nil {
 		t.Fatal(err)
 	}
 	if err := r.Connect(ctx); err != nil {
@@ -99,9 +120,11 @@ func Test_rkv_configure(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
+			b := atomic.Bool{}
 			rc := &Store{
-				opts: tt.fields.options,
-				cli:  tt.fields.Client,
+				opts:      tt.fields.options,
+				cli:       tt.fields.Client,
+				connected: &b,
 			}
 			err := rc.configure()
 			if (err != nil) != tt.wantErr {

stats.go

@@ -33,20 +33,17 @@ func (r *Store) statsMeter() {
 		ticker := time.NewTicker(meter.DefaultMeterStatsInterval)
 		defer ticker.Stop()
 
-		for {
-			select {
-			case <-ticker.C:
-				if st == nil {
-					return
-				}
-				stats := st.PoolStats()
-				r.opts.Meter.Counter(PoolHitsTotal).Set(uint64(stats.Hits))
-				r.opts.Meter.Counter(PoolMissesTotal).Set(uint64(stats.Misses))
-				r.opts.Meter.Counter(PoolTimeoutTotal).Set(uint64(stats.Timeouts))
-				r.opts.Meter.Counter(PoolConnTotalCurrent).Set(uint64(stats.TotalConns))
-				r.opts.Meter.Counter(PoolConnIdleCurrent).Set(uint64(stats.IdleConns))
-				r.opts.Meter.Counter(PoolConnStaleTotal).Set(uint64(stats.StaleConns))
+		for _ = range ticker.C {
+			if st == nil {
+				return
 			}
+			stats := st.PoolStats()
+			r.opts.Meter.Counter(PoolHitsTotal).Set(uint64(stats.Hits))
+			r.opts.Meter.Counter(PoolMissesTotal).Set(uint64(stats.Misses))
+			r.opts.Meter.Counter(PoolTimeoutTotal).Set(uint64(stats.Timeouts))
+			r.opts.Meter.Counter(PoolConnTotalCurrent).Set(uint64(stats.TotalConns))
+			r.opts.Meter.Counter(PoolConnIdleCurrent).Set(uint64(stats.IdleConns))
+			r.opts.Meter.Counter(PoolConnStaleTotal).Set(uint64(stats.StaleConns))
 		}
 	}()
 }

tracer.go

@@ -73,7 +73,7 @@ func (h *tracingHook) ProcessHook(hook goredis.ProcessHook) goredis.ProcessHook
 		case "cluster slots":
 			break
 		default:
-			_, span := h.tr.Start(ctx, "goredis.process", append(h.opts, tracer.WithSpanLabels("db.statement", cmdString))...)
+			_, span := h.tr.Start(ctx, "sdk.database", append(h.opts, tracer.WithSpanLabels("db.statement", cmdString))...)
 			defer func() {
 				recordError(span, err)
 				span.Finish()
@@ -91,11 +91,11 @@ func (h *tracingHook) ProcessPipelineHook(hook goredis.ProcessPipelineHook) gore
 		_, cmdsString := rediscmd.CmdsString(cmds)
 
 		opts := append(h.opts, tracer.WithSpanLabels(
-			"db.goredis.num_cmd", strconv.Itoa(len(cmds)),
+			"db.database.num_cmd", strconv.Itoa(len(cmds)),
 			"db.statement", cmdsString,
 		))
 
-		_, span := h.tr.Start(ctx, "goredis.process_pipeline", opts...)
+		_, span := h.tr.Start(ctx, "sdk.database", opts...)
 		defer span.Finish()
 
 		err := hook(ctx, cmds)