add store.Watcher interface (#123)

Signed-off-by: Vasiliy Tolstov <v.tolstov@unistack.org>
Reviewed-on: #123
Co-authored-by: Vasiliy Tolstov <v.tolstov@unistack.org>
Co-committed-by: Vasiliy Tolstov <v.tolstov@unistack.org>

event.go

@@ -0,0 +1,86 @@
+package redis
+
+import (
+	"context"
+	"errors"
+	"net"
+	"time"
+
+	goredis "github.com/redis/go-redis/v9"
+	"go.unistack.org/micro/v3/store"
+)
+
+type eventHook struct {
+	s *Store
+}
+
+var _ goredis.Hook = (*eventHook)(nil)
+
+func newEventHook(s *Store) *eventHook {
+	return &eventHook{s: s}
+}
+
+func (h *eventHook) DialHook(hook goredis.DialHook) goredis.DialHook {
+	return func(ctx context.Context, network, addr string) (net.Conn, error) {
+		conn, err := hook(ctx, network, addr)
+		if err != nil {
+			if !isRedisError(err) {
+				if h.s.connected.CompareAndSwap(1, 0) {
+					h.s.sendEvent(&event{ts: time.Now(), err: err, t: store.EventTypeDisconnect})
+				}
+			} else {
+				h.s.connected.Store(1)
+			}
+		} else {
+			if h.s.connected.CompareAndSwap(0, 1) {
+				h.s.sendEvent(&event{ts: time.Now(), err: err, t: store.EventTypeConnect})
+			}
+		}
+		return conn, err
+	}
+}
+
+func (h *eventHook) ProcessHook(hook goredis.ProcessHook) goredis.ProcessHook {
+	return func(ctx context.Context, cmd goredis.Cmder) error {
+		err := hook(ctx, cmd)
+		if err != nil {
+			if !isRedisError(err) {
+				if h.s.connected.CompareAndSwap(1, 0) {
+					h.s.sendEvent(&event{ts: time.Now(), err: err, t: store.EventTypeDisconnect})
+				}
+			} else {
+				h.s.connected.Store(1)
+			}
+		} else {
+			if h.s.connected.CompareAndSwap(0, 1) {
+				h.s.sendEvent(&event{ts: time.Now(), err: err, t: store.EventTypeConnect})
+			}
+		}
+		return err
+	}
+}
+
+func (h *eventHook) ProcessPipelineHook(hook goredis.ProcessPipelineHook) goredis.ProcessPipelineHook {
+	return func(ctx context.Context, cmds []goredis.Cmder) error {
+		err := hook(ctx, cmds)
+		if err != nil {
+			if !isRedisError(err) {
+				if h.s.connected.CompareAndSwap(1, 0) {
+					h.s.sendEvent(&event{ts: time.Now(), err: err, t: store.EventTypeDisconnect})
+				}
+			} else {
+				h.s.connected.Store(1)
+			}
+		} else {
+			if h.s.connected.CompareAndSwap(0, 1) {
+				h.s.sendEvent(&event{ts: time.Now(), err: err, t: store.EventTypeConnect})
+			}
+		}
+		return err
+	}
+}
+
+func isRedisError(err error) bool {
+	var rerr goredis.Error
+	return errors.As(err, &rerr)
+}

go.mod

@@ -1,24 +1,19 @@
 module go.unistack.org/micro-store-redis/v3
 
-go 1.21
+go 1.22
 
 toolchain go1.22.4
 
 require (
-	github.com/redis/go-redis/extra/rediscmd/v9 v9.5.3
-	github.com/redis/go-redis/v9 v9.6.1
-	go.unistack.org/micro/v3 v3.10.95
+	github.com/redis/go-redis/extra/rediscmd/v9 v9.7.0
+	github.com/redis/go-redis/v9 v9.7.0
+	go.unistack.org/micro/v3 v3.10.108
 )
 
 require (
-	dario.cat/mergo v1.0.0 // indirect
 	github.com/cespare/xxhash/v2 v2.3.0 // indirect
 	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
-	github.com/golang/protobuf v1.5.3 // indirect
-	github.com/google/uuid v1.3.0 // indirect
-	github.com/silas/dag v0.0.0-20220518035006-a7e85ada93c5 // indirect
+	github.com/google/go-cmp v0.6.0 // indirect
 	go.unistack.org/micro-proto/v3 v3.4.1 // indirect
-	google.golang.org/genproto/googleapis/rpc v0.0.0-20230526203410-71b5a4ffd15e // indirect
-	google.golang.org/grpc v1.57.0 // indirect
-	google.golang.org/protobuf v1.34.2 // indirect
+	google.golang.org/protobuf v1.35.2 // indirect
 )

go.sum

@@ -1,45 +1,20 @@
-dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk=
-dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk=
 github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
 github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=
 github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
 github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0=
-github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
-github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/cespare/xxhash/v2 v2.3.0 h1:UL815xU9SqsFlibzuggzjXhog7bL6oX9BbNZnL2UFvs=
 github.com/cespare/xxhash/v2 v2.3.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
 github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
-github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
-github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg=
-github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
-github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
-github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/redis/go-redis/extra/rediscmd/v9 v9.5.3 h1:1/BDligzCa40GTllkDnY3Y5DTHuKCONbB2JcRyIfl20=
-github.com/redis/go-redis/extra/rediscmd/v9 v9.5.3/go.mod h1:3dZmcLn3Qw6FLlWASn1g4y+YO9ycEFUOM+bhBmzLVKQ=
-github.com/redis/go-redis/v9 v9.5.3 h1:fOAp1/uJG+ZtcITgZOfYFmTKPE7n4Vclj1wZFgRciUU=
-github.com/redis/go-redis/v9 v9.5.3/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M=
-github.com/redis/go-redis/v9 v9.6.1 h1:HHDteefn6ZkTtY5fGUE8tj8uy85AHk6zP7CpzIAM0y4=
-github.com/redis/go-redis/v9 v9.6.1/go.mod h1:0C0c6ycQsdpVNQpxb1njEQIqkx5UcsM8FJCQLgE9+RA=
-github.com/silas/dag v0.0.0-20220518035006-a7e85ada93c5 h1:G/FZtUu7a6NTWl3KUHMV9jkLAh/Rvtf03NWMHaEDl+E=
-github.com/silas/dag v0.0.0-20220518035006-a7e85ada93c5/go.mod h1:7RTUFBdIRC9nZ7/3RyRNH1bdqIShrDejd1YbLwgPS+I=
+github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
+github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
+github.com/redis/go-redis/extra/rediscmd/v9 v9.7.0 h1:BIx9TNZH/Jsr4l1i7VVxnV0JPiwYj8qyrHyuL0fGZrk=
+github.com/redis/go-redis/extra/rediscmd/v9 v9.7.0/go.mod h1:eTg/YQtGYAZD5r3DlGlJptJ45AHA+/G+2NPn30PKzik=
+github.com/redis/go-redis/v9 v9.7.0 h1:HhLSs+B6O021gwzl+locl0zEDnyNkxMtf/Z3NNBMa9E=
+github.com/redis/go-redis/v9 v9.7.0/go.mod h1:f6zhXITC7JUJIlPEiBOTXxJgPLdZcA93GewI7inzyWw=
 go.unistack.org/micro-proto/v3 v3.4.1 h1:UTjLSRz2YZuaHk9iSlVqqsA50JQNAEK2ZFboGqtEa9Q=
 go.unistack.org/micro-proto/v3 v3.4.1/go.mod h1:okx/cnOhzuCX0ggl/vToatbCupi0O44diiiLLsZ93Zo=
-go.unistack.org/micro/v3 v3.10.80 h1:A0zWNoM9MOcMg9gdFFgVkgbT3uSYVIINhuvumX9nP2o=
-go.unistack.org/micro/v3 v3.10.80/go.mod h1:erMgt3Bl7vQQ0e9UpQyR5NlLiZ9pKeEJ9+1tfYFaqUg=
-go.unistack.org/micro/v3 v3.10.94 h1:t9J+vw6rNOQdEPFPeUvhyxlUyL+LEt9dTV/bjSEaxMI=
-go.unistack.org/micro/v3 v3.10.94/go.mod h1:erMgt3Bl7vQQ0e9UpQyR5NlLiZ9pKeEJ9+1tfYFaqUg=
-go.unistack.org/micro/v3 v3.10.95 h1:KQse+ZpntbuzA8cH6Pz7CdvWYVdKzcR34gjnkDrHiso=
-go.unistack.org/micro/v3 v3.10.95/go.mod h1:erMgt3Bl7vQQ0e9UpQyR5NlLiZ9pKeEJ9+1tfYFaqUg=
-golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20230526203410-71b5a4ffd15e h1:NumxXLPfHSndr3wBBdeKiVHjGVFzi9RX2HwwQke94iY=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20230526203410-71b5a4ffd15e/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA=
-google.golang.org/grpc v1.57.0 h1:kfzNeI/klCGD2YPMUlaGNT3pxvYfga7smW3Vth8Zsiw=
-google.golang.org/grpc v1.57.0/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo=
-google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
-google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
-google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg=
-google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw=
-gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
-gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+go.unistack.org/micro/v3 v3.10.108 h1:3L7SkilMVLtH8y3pQIPtr3jjQYrf0AMv1oAkoL3nFkE=
+go.unistack.org/micro/v3 v3.10.108/go.mod h1:YzMldzHN9Ei+zy5t/Psu7RUWDZwUfrNYiStSQtTz90g=
+google.golang.org/protobuf v1.35.2 h1:8Ar7bF+apOIoThw1EdZl0p1oWvMqTHmpA2fRTyZO8io=
+google.golang.org/protobuf v1.35.2/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE=

options.go

@@ -61,7 +61,7 @@ func NewOptions(opts ...Option) Options {
 			labelName, options.RedisName),
 	)
 
-	options.Logger = options.Logger.Clone(logger.WithCallerSkipCount(1))
+	options.Logger = options.Logger.Clone(logger.WithAddCallerSkipCount(1))
 
 	return options
 }

redis.go

@@ -2,20 +2,25 @@ package redis
 
 import (
 	"context"
+	"errors"
 	"reflect"
 	"strings"
+	"sync"
+	"sync/atomic"
 	"time"
 
 	goredis "github.com/redis/go-redis/v9"
 	"go.unistack.org/micro/v3/semconv"
 	"go.unistack.org/micro/v3/store"
+	"go.unistack.org/micro/v3/util/id"
 	pool "go.unistack.org/micro/v3/util/xpool"
 )
 
 var (
-	DefaultPathSeparator = "/"
-
-	DefaultUniversalOptions = &goredis.UniversalOptions{
+	_                       store.Store = (*Store)(nil)
+	_                       store.Event = (*event)(nil)
+	sendEventTime                       = 10 * time.Millisecond
+	DefaultUniversalOptions             = &goredis.UniversalOptions{
 		Username:        "",
 		Password:        "", // no password set
 		MaxRetries:      2,
@@ -54,19 +59,30 @@ var (
 )
 
 type Store struct {
-	opts store.Options
-	cli  goredis.UniversalClient
-	done chan struct{}
-	pool *pool.StringsPool
+	cli       goredis.UniversalClient
+	pool      *pool.StringsPool
+	connected *atomic.Uint32
+	opts      store.Options
+	watchers  map[string]*watcher
+	mu        sync.RWMutex
 }
 
 func (r *Store) Connect(ctx context.Context) error {
+	if r.connected.Load() == 1 {
+		return nil
+	}
 	if r.cli == nil {
 		return store.ErrNotConnected
 	}
-	err := r.cli.Ping(ctx).Err()
-	setSpanError(ctx, err)
-	return err
+	if r.opts.LazyConnect {
+		return nil
+	}
+	if err := r.cli.Ping(ctx).Err(); err != nil {
+		setSpanError(ctx, err)
+		return err
+	}
+	r.connected.Store(1)
+	return nil
 }
 
 func (r *Store) Init(opts ...store.Option) error {
@@ -101,23 +117,26 @@ func (r *Store) ClusterClient() *goredis.ClusterClient {
 }
 
 func (r *Store) Disconnect(ctx context.Context) error {
-	var err error
-	select {
-	case <-r.done:
-		return err
-	default:
-		if r.cli != nil {
-			err = r.cli.Close()
-		}
-		close(r.done)
-		return err
+	if r.connected.Load() == 0 {
+		return nil
 	}
+
+	if r.cli != nil {
+		if err := r.cli.Close(); err != nil {
+			return err
+		}
+	}
+
+	r.connected.Store(1)
+	return nil
 }
 
 func (r *Store) Exists(ctx context.Context, key string, opts ...store.ExistsOption) error {
 	b := r.pool.Get()
 	defer r.pool.Put(b)
 	options := store.NewExistsOptions(opts...)
+	labels := make([]string, 0, 6)
+	labels = append(labels, "name", options.Name, "statement", "exists")
 
 	timeout := r.opts.Timeout
 	if options.Timeout > 0 {
@@ -132,21 +151,22 @@ func (r *Store) Exists(ctx context.Context, key string, opts ...store.ExistsOpti
 
 	rkey := r.getKey(b, r.opts.Namespace, options.Namespace, key)
 
-	r.opts.Meter.Counter(semconv.StoreRequestInflight, "name", options.Name).Inc()
+	r.opts.Meter.Counter(semconv.StoreRequestInflight, labels...).Inc()
 	ts := time.Now()
 	val, err := r.cli.Exists(ctx, rkey).Result()
 	setSpanError(ctx, err)
 	te := time.Since(ts)
-	r.opts.Meter.Counter(semconv.StoreRequestInflight, "name", options.Name).Dec()
-	r.opts.Meter.Summary(semconv.StoreRequestLatencyMicroseconds, "name", options.Name).Update(te.Seconds())
-	r.opts.Meter.Histogram(semconv.StoreRequestDurationSeconds, "name", options.Name).Update(te.Seconds())
-	if err == goredis.Nil || (err == nil && val == 0) {
-		r.opts.Meter.Counter(semconv.StoreRequestTotal, "name", options.Name, "status", "miss").Inc()
+	r.opts.Meter.Counter(semconv.StoreRequestInflight, labels...).Dec()
+
+	r.opts.Meter.Summary(semconv.StoreRequestLatencyMicroseconds, labels...).Update(te.Seconds())
+	r.opts.Meter.Histogram(semconv.StoreRequestDurationSeconds, labels...).Update(te.Seconds())
+	if errors.Is(err, goredis.Nil) || (err == nil && val == 0) {
+		r.opts.Meter.Counter(semconv.StoreRequestTotal, append(labels, "status", "miss")...).Inc()
 		return store.ErrNotFound
 	} else if err == nil {
-		r.opts.Meter.Counter(semconv.StoreRequestTotal, "name", options.Name, "status", "hit").Inc()
-	} else if err != nil {
-		r.opts.Meter.Counter(semconv.StoreRequestTotal, "name", options.Name, "status", "failure").Inc()
+		r.opts.Meter.Counter(semconv.StoreRequestTotal, append(labels, "status", "hit")...).Inc()
+	} else {
+		r.opts.Meter.Counter(semconv.StoreRequestTotal, append(labels, "status", "failure")...).Inc()
 		return err
 	}
 
@@ -158,6 +178,8 @@ func (r *Store) Read(ctx context.Context, key string, val interface{}, opts ...s
 	defer r.pool.Put(b)
 
 	options := store.NewReadOptions(opts...)
+	labels := make([]string, 0, 6)
+	labels = append(labels, "name", options.Name, "statement", "read")
 
 	timeout := r.opts.Timeout
 	if options.Timeout > 0 {
@@ -172,21 +194,21 @@ func (r *Store) Read(ctx context.Context, key string, val interface{}, opts ...s
 
 	rkey := r.getKey(b, r.opts.Namespace, options.Namespace, key)
 
-	r.opts.Meter.Counter(semconv.StoreRequestInflight, "name", options.Name).Inc()
+	r.opts.Meter.Counter(semconv.StoreRequestInflight, labels...).Inc()
 	ts := time.Now()
 	buf, err := r.cli.Get(ctx, rkey).Bytes()
 	setSpanError(ctx, err)
 	te := time.Since(ts)
-	r.opts.Meter.Counter(semconv.StoreRequestInflight, "name", options.Name).Dec()
-	r.opts.Meter.Summary(semconv.StoreRequestLatencyMicroseconds, "name", options.Name).Update(te.Seconds())
-	r.opts.Meter.Histogram(semconv.StoreRequestDurationSeconds, "name", options.Name).Update(te.Seconds())
-	if err == goredis.Nil || (err == nil && buf == nil) {
-		r.opts.Meter.Counter(semconv.StoreRequestTotal, "name", options.Name, "status", "miss").Inc()
+	r.opts.Meter.Counter(semconv.StoreRequestInflight, labels...).Dec()
+	r.opts.Meter.Summary(semconv.StoreRequestLatencyMicroseconds, labels...).Update(te.Seconds())
+	r.opts.Meter.Histogram(semconv.StoreRequestDurationSeconds, labels...).Update(te.Seconds())
+	if errors.Is(err, goredis.Nil) || (err == nil && buf == nil) {
+		r.opts.Meter.Counter(semconv.StoreRequestTotal, append(labels, "status", "miss")...).Inc()
 		return store.ErrNotFound
 	} else if err == nil {
-		r.opts.Meter.Counter(semconv.StoreRequestTotal, "name", options.Name, "status", "hit").Inc()
+		r.opts.Meter.Counter(semconv.StoreRequestTotal, append(labels, "status", "hit")...).Inc()
 	} else if err != nil {
-		r.opts.Meter.Counter(semconv.StoreRequestTotal, "name", options.Name, "status", "failure").Inc()
+		r.opts.Meter.Counter(semconv.StoreRequestTotal, append(labels, "status", "failure")...).Inc()
 		return err
 	}
 
@@ -305,6 +327,8 @@ func (r *Store) MRead(ctx context.Context, keys []string, vals interface{}, opts
 
 func (r *Store) MDelete(ctx context.Context, keys []string, opts ...store.DeleteOption) error {
 	options := store.NewDeleteOptions(opts...)
+	labels := make([]string, 0, 6)
+	labels = append(labels, "name", options.Name, "statement", "delete")
 
 	timeout := r.opts.Timeout
 	if options.Timeout > 0 {
@@ -329,7 +353,7 @@ func (r *Store) MDelete(ctx context.Context, keys []string, opts ...store.Delete
 		}
 	}
 
-	r.opts.Meter.Counter(semconv.StoreRequestInflight, "name", options.Name).Inc()
+	r.opts.Meter.Counter(semconv.StoreRequestInflight, labels...).Inc()
 	ts := time.Now()
 	var err error
 	if r.opts.Namespace != "" || options.Namespace != "" {
@@ -342,16 +366,16 @@ func (r *Store) MDelete(ctx context.Context, keys []string, opts ...store.Delete
 	}
 	setSpanError(ctx, err)
 	te := time.Since(ts)
-	r.opts.Meter.Counter(semconv.StoreRequestInflight, "name", options.Name).Dec()
-	r.opts.Meter.Summary(semconv.StoreRequestLatencyMicroseconds, "name", options.Name).Update(te.Seconds())
-	r.opts.Meter.Histogram(semconv.StoreRequestDurationSeconds, "name", options.Name).Update(te.Seconds())
+	r.opts.Meter.Counter(semconv.StoreRequestInflight, labels...).Dec()
+	r.opts.Meter.Summary(semconv.StoreRequestLatencyMicroseconds, labels...).Update(te.Seconds())
+	r.opts.Meter.Histogram(semconv.StoreRequestDurationSeconds, labels...).Update(te.Seconds())
 	if err == goredis.Nil {
-		r.opts.Meter.Counter(semconv.StoreRequestTotal, "name", options.Name, "status", "miss").Inc()
+		r.opts.Meter.Counter(semconv.StoreRequestTotal, append(labels, "status", "miss")...).Inc()
 		return store.ErrNotFound
 	} else if err == nil {
-		r.opts.Meter.Counter(semconv.StoreRequestTotal, "name", options.Name, "status", "hit").Inc()
+		r.opts.Meter.Counter(semconv.StoreRequestTotal, append(labels, "status", "hit")...).Inc()
 	} else if err != nil {
-		r.opts.Meter.Counter(semconv.StoreRequestTotal, "name", options.Name, "status", "failure").Inc()
+		r.opts.Meter.Counter(semconv.StoreRequestTotal, append(labels, "status", "failure")...).Inc()
 		return err
 	}
 
@@ -363,6 +387,8 @@ func (r *Store) Delete(ctx context.Context, key string, opts ...store.DeleteOpti
 	defer r.pool.Put(b)
 
 	options := store.NewDeleteOptions(opts...)
+	labels := make([]string, 0, 6)
+	labels = append(labels, "name", options.Name, "statement", "delete")
 
 	timeout := r.opts.Timeout
 	if options.Timeout > 0 {
@@ -375,21 +401,22 @@ func (r *Store) Delete(ctx context.Context, key string, opts ...store.DeleteOpti
 		defer cancel()
 	}
 
-	r.opts.Meter.Counter(semconv.StoreRequestInflight, "name", options.Name).Inc()
+	r.opts.Meter.Counter(semconv.StoreRequestInflight, labels...).Inc()
 	ts := time.Now()
 	err := r.cli.Del(ctx, r.getKey(b, r.opts.Namespace, options.Namespace, key)).Err()
 	te := time.Since(ts)
 	setSpanError(ctx, err)
-	r.opts.Meter.Counter(semconv.StoreRequestInflight, "name", options.Name).Dec()
-	r.opts.Meter.Summary(semconv.StoreRequestLatencyMicroseconds, "name", options.Name).Update(te.Seconds())
-	r.opts.Meter.Histogram(semconv.StoreRequestDurationSeconds, "name", options.Name).Update(te.Seconds())
-	if err == goredis.Nil {
-		r.opts.Meter.Counter(semconv.StoreRequestTotal, "name", options.Name, "status", "miss").Inc()
+
+	r.opts.Meter.Counter(semconv.StoreRequestInflight, labels...).Dec()
+	r.opts.Meter.Summary(semconv.StoreRequestLatencyMicroseconds, labels...).Update(te.Seconds())
+	r.opts.Meter.Histogram(semconv.StoreRequestDurationSeconds, labels...).Update(te.Seconds())
+	if errors.Is(err, goredis.Nil) {
+		r.opts.Meter.Counter(semconv.StoreRequestTotal, append(labels, "status", "miss")...).Inc()
 		return store.ErrNotFound
 	} else if err == nil {
-		r.opts.Meter.Counter(semconv.StoreRequestTotal, "name", options.Name, "status", "hit").Inc()
+		r.opts.Meter.Counter(semconv.StoreRequestTotal, append(labels, "status", "hit")...).Inc()
 	} else if err != nil {
-		r.opts.Meter.Counter(semconv.StoreRequestTotal, "name", options.Name, "status", "failure").Inc()
+		r.opts.Meter.Counter(semconv.StoreRequestTotal, append(labels, "status", "failure")...).Inc()
 		return err
 	}
 
@@ -398,6 +425,8 @@ func (r *Store) Delete(ctx context.Context, key string, opts ...store.DeleteOpti
 
 func (r *Store) MWrite(ctx context.Context, keys []string, vals []interface{}, opts ...store.WriteOption) error {
 	options := store.NewWriteOptions(opts...)
+	labels := make([]string, 0, 6)
+	labels = append(labels, "name", options.Name, "statement", "write")
 
 	timeout := r.opts.Timeout
 	if options.Timeout > 0 {
@@ -431,7 +460,7 @@ func (r *Store) MWrite(ctx context.Context, keys []string, vals []interface{}, o
 		}
 	}
 
-	r.opts.Meter.Counter(semconv.StoreRequestInflight, "name", options.Name).Inc()
+	r.opts.Meter.Counter(semconv.StoreRequestInflight, labels...).Inc()
 
 	pipeliner := func(pipe goredis.Pipeliner) error {
 		for idx := 0; idx < len(kvs); idx += 2 {
@@ -451,27 +480,27 @@ func (r *Store) MWrite(ctx context.Context, keys []string, vals []interface{}, o
 	te := time.Since(ts)
 	setSpanError(ctx, err)
 
-	r.opts.Meter.Counter(semconv.StoreRequestInflight, "name", options.Name).Dec()
-	r.opts.Meter.Summary(semconv.StoreRequestLatencyMicroseconds, "name", options.Name).Update(te.Seconds())
-	r.opts.Meter.Histogram(semconv.StoreRequestDurationSeconds, "name", options.Name).Update(te.Seconds())
+	r.opts.Meter.Counter(semconv.StoreRequestInflight, labels...).Dec()
+	r.opts.Meter.Summary(semconv.StoreRequestLatencyMicroseconds, labels...).Update(te.Seconds())
+	r.opts.Meter.Histogram(semconv.StoreRequestDurationSeconds, labels...).Update(te.Seconds())
 	if err == goredis.Nil {
-		r.opts.Meter.Counter(semconv.StoreRequestTotal, "name", options.Name, "status", "miss").Inc()
+		r.opts.Meter.Counter(semconv.StoreRequestTotal, append(labels, "status", "miss")...).Inc()
 		return store.ErrNotFound
 	} else if err == nil {
-		r.opts.Meter.Counter(semconv.StoreRequestTotal, "name", options.Name, "status", "hit").Inc()
+		r.opts.Meter.Counter(semconv.StoreRequestTotal, append(labels, "status", "hit")...).Inc()
 	} else if err != nil {
-		r.opts.Meter.Counter(semconv.StoreRequestTotal, "name", options.Name, "status", "failure").Inc()
+		r.opts.Meter.Counter(semconv.StoreRequestTotal, append(labels, "status", "failure")...).Inc()
 		return err
 	}
 
 	for _, cmd := range cmds {
 		if err = cmd.Err(); err != nil {
 			if err == goredis.Nil {
-				r.opts.Meter.Counter(semconv.StoreRequestTotal, "name", options.Name, "status", "miss").Inc()
+				r.opts.Meter.Counter(semconv.StoreRequestTotal, append(labels, "status", "miss")...).Inc()
 				return store.ErrNotFound
 			}
 			setSpanError(ctx, err)
-			r.opts.Meter.Counter(semconv.StoreRequestTotal, "name", options.Name, "status", "failure").Inc()
+			r.opts.Meter.Counter(semconv.StoreRequestTotal, append(labels, "status", "failure")...).Inc()
 			return err
 		}
 	}
@@ -484,6 +513,8 @@ func (r *Store) Write(ctx context.Context, key string, val interface{}, opts ...
 	defer r.pool.Put(b)
 
 	options := store.NewWriteOptions(opts...)
+	labels := make([]string, 0, 6)
+	labels = append(labels, "name", options.Name, "statement", "write")
 
 	timeout := r.opts.Timeout
 	if options.Timeout > 0 {
@@ -512,22 +543,22 @@ func (r *Store) Write(ctx context.Context, key string, val interface{}, opts ...
 		}
 	}
 
-	r.opts.Meter.Counter(semconv.StoreRequestInflight, "name", options.Name).Inc()
+	r.opts.Meter.Counter(semconv.StoreRequestInflight, labels...).Inc()
 	ts := time.Now()
 	err := r.cli.Set(ctx, rkey, buf, options.TTL).Err()
 	te := time.Since(ts)
 	setSpanError(ctx, err)
 
-	r.opts.Meter.Counter(semconv.StoreRequestInflight, "name", options.Name).Dec()
-	r.opts.Meter.Summary(semconv.StoreRequestLatencyMicroseconds, "name", options.Name).Update(te.Seconds())
-	r.opts.Meter.Histogram(semconv.StoreRequestDurationSeconds, "name", options.Name).Update(te.Seconds())
-	if err == goredis.Nil {
-		r.opts.Meter.Counter(semconv.StoreRequestTotal, "name", options.Name, "status", "miss").Inc()
+	r.opts.Meter.Counter(semconv.StoreRequestInflight, labels...).Dec()
+	r.opts.Meter.Summary(semconv.StoreRequestLatencyMicroseconds, labels...).Update(te.Seconds())
+	r.opts.Meter.Histogram(semconv.StoreRequestDurationSeconds, labels...).Update(te.Seconds())
+	if errors.Is(err, goredis.Nil) {
+		r.opts.Meter.Counter(semconv.StoreRequestTotal, append(labels, "status", "miss")...).Inc()
 		return store.ErrNotFound
 	} else if err == nil {
-		r.opts.Meter.Counter(semconv.StoreRequestTotal, "name", options.Name, "status", "hit").Inc()
-	} else if err != nil {
-		r.opts.Meter.Counter(semconv.StoreRequestTotal, "name", options.Name, "status", "failure").Inc()
+		r.opts.Meter.Counter(semconv.StoreRequestTotal, append(labels, "status", "hit")...).Inc()
+	} else {
+		r.opts.Meter.Counter(semconv.StoreRequestTotal, append(labels, "status", "failure")...).Inc()
 		return err
 	}
 
@@ -539,6 +570,9 @@ func (r *Store) List(ctx context.Context, opts ...store.ListOption) ([]string, e
 	defer r.pool.Put(b)
 
 	options := store.NewListOptions(opts...)
+	labels := make([]string, 0, 6)
+	labels = append(labels, "name", options.Name, "statement", "list")
+
 	if len(options.Namespace) == 0 {
 		options.Namespace = r.opts.Namespace
 	}
@@ -560,7 +594,7 @@ func (r *Store) List(ctx context.Context, opts ...store.ListOption) ([]string, e
 	}
 
 	// TODO: add support for prefix/suffix/limit
-	r.opts.Meter.Counter(semconv.StoreRequestInflight, "name", options.Name).Inc()
+	r.opts.Meter.Counter(semconv.StoreRequestInflight, labels...).Inc()
 	ts := time.Now()
 	var keys []string
 	var err error
@@ -580,16 +614,16 @@ func (r *Store) List(ctx context.Context, opts ...store.ListOption) ([]string, e
 	te := time.Since(ts)
 	setSpanError(ctx, err)
 
-	r.opts.Meter.Counter(semconv.StoreRequestInflight, "name", options.Name).Dec()
-	r.opts.Meter.Summary(semconv.StoreRequestLatencyMicroseconds, "name", options.Name).Update(te.Seconds())
-	r.opts.Meter.Histogram(semconv.StoreRequestDurationSeconds, "name", options.Name).Update(te.Seconds())
+	r.opts.Meter.Counter(semconv.StoreRequestInflight, labels...).Dec()
+	r.opts.Meter.Summary(semconv.StoreRequestLatencyMicroseconds, labels...).Update(te.Seconds())
+	r.opts.Meter.Histogram(semconv.StoreRequestDurationSeconds, labels...).Update(te.Seconds())
 	if err == goredis.Nil {
-		r.opts.Meter.Counter(semconv.StoreRequestTotal, "name", options.Name, "status", "miss").Inc()
+		r.opts.Meter.Counter(semconv.StoreRequestTotal, append(labels, "status", "miss")...).Inc()
 		return nil, store.ErrNotFound
 	} else if err == nil {
-		r.opts.Meter.Counter(semconv.StoreRequestTotal, "name", options.Name, "status", "hit").Inc()
+		r.opts.Meter.Counter(semconv.StoreRequestTotal, append(labels, "status", "git")...).Inc()
 	} else if err != nil {
-		r.opts.Meter.Counter(semconv.StoreRequestTotal, "name", options.Name, "status", "failure").Inc()
+		r.opts.Meter.Counter(semconv.StoreRequestTotal, append(labels, "status", "failure")...).Inc()
 		return nil, err
 	}
 
@@ -621,16 +655,21 @@ func (r *Store) String() string {
 }
 
 func NewStore(opts ...store.Option) *Store {
-	return &Store{done: make(chan struct{}), opts: store.NewOptions(opts...)}
+	b := atomic.Uint32{}
+	return &Store{
+		opts:      store.NewOptions(opts...),
+		connected: &b,
+		watchers:  make(map[string]*watcher),
+	}
 }
 
 func (r *Store) configure() error {
-	var universalOptions *goredis.UniversalOptions
-
 	if r.cli != nil && r.opts.Context == nil {
 		return nil
 	}
 
+	universalOptions := DefaultUniversalOptions
+
 	if r.opts.Context != nil {
 		if o, ok := r.opts.Context.Value(configKey{}).(*goredis.Options); ok {
 			universalOptions.Addrs = []string{o.Addr}
@@ -704,10 +743,6 @@ func (r *Store) configure() error {
 		}
 	}
 
-	if universalOptions == nil {
-		universalOptions = DefaultUniversalOptions
-	}
-
 	if len(r.opts.Addrs) > 0 {
 		universalOptions.Addrs = r.opts.Addrs
 	} else {
@@ -716,6 +751,7 @@ func (r *Store) configure() error {
 
 	r.cli = goredis.NewUniversalClient(universalOptions)
 	setTracing(r.cli, r.opts.Tracer)
+	r.cli.AddHook(newEventHook(r))
 
 	r.pool = pool.NewStringsPool(50)
 
@@ -730,8 +766,99 @@ func (r *Store) getKey(b *strings.Builder, mainNamespace string, opNamespace str
 	}
 	if opNamespace != "" {
 		b.WriteString(opNamespace)
-		b.WriteString(DefaultPathSeparator)
+		b.WriteString(r.opts.Separator)
 	}
 	b.WriteString(key)
 	return b.String()
 }
+
+func (r *Store) Watch(ctx context.Context, opts ...store.WatchOption) (store.Watcher, error) {
+	id, err := id.New()
+	if err != nil {
+		return nil, err
+	}
+	wo, err := store.NewWatchOptions(opts...)
+	if err != nil {
+		return nil, err
+	}
+	// construct the watcher
+	w := &watcher{
+		exit: make(chan bool),
+		ch:   make(chan store.Event),
+		id:   id,
+		opts: wo,
+	}
+
+	r.mu.Lock()
+	r.watchers[w.id] = w
+	r.mu.Unlock()
+
+	return w, nil
+}
+
+func (r *Store) sendEvent(e store.Event) {
+	r.mu.RLock()
+	watchers := make([]*watcher, 0, len(r.watchers))
+	for _, w := range r.watchers {
+		watchers = append(watchers, w)
+	}
+	r.mu.RUnlock()
+	for _, w := range watchers {
+		select {
+		case <-w.exit:
+			r.mu.Lock()
+			delete(r.watchers, w.id)
+			r.mu.Unlock()
+		default:
+			select {
+			case w.ch <- e:
+			case <-time.After(sendEventTime):
+			}
+		}
+	}
+}
+
+type watcher struct {
+	ch   chan store.Event
+	exit chan bool
+	opts store.WatchOptions
+	id   string
+}
+
+func (w *watcher) Next() (store.Event, error) {
+	for {
+		select {
+		case e := <-w.ch:
+			return e, nil
+		case <-w.exit:
+			return nil, store.ErrWatcherStopped
+		}
+	}
+}
+
+func (w *watcher) Stop() {
+	select {
+	case <-w.exit:
+		return
+	default:
+		close(w.exit)
+	}
+}
+
+type event struct {
+	ts  time.Time
+	t   store.EventType
+	err error
+}
+
+func (e *event) Error() error {
+	return e.err
+}
+
+func (e *event) Timestamp() time.Time {
+	return e.ts
+}
+
+func (e *event) Type() store.EventType {
+	return e.t
+}

redis_test.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"context"
 	"os"
+	"sync/atomic"
 	"testing"
 	"time"
 
@@ -12,6 +13,26 @@ import (
 	"go.unistack.org/micro/v3/tracer"
 )
 
+func TestLazyConnect(t *testing.T) {
+	ctx := context.Background()
+	var err error
+
+	r := NewStore()
+
+	if err = r.Init(); err != nil {
+		t.Fatal(err)
+	}
+	if err = r.Connect(ctx); err != nil {
+		t.Logf("connect failed %v", err)
+	}
+
+	for {
+		if err = r.Write(ctx, "mykey", "myval"); err != nil {
+			t.Logf("failed to write %v", err)
+		}
+	}
+}
+
 func TestKeepTTL(t *testing.T) {
 	ctx := context.Background()
 
@@ -20,7 +41,7 @@ func TestKeepTTL(t *testing.T) {
 	}
 	r := NewStore(store.Addrs(os.Getenv("STORE_NODES")))
 
-	if err := r.Init(); err != nil {
+	if err := r.Init(store.LazyConnect(true)); err != nil {
 		t.Fatal(err)
 	}
 	if err := r.Connect(ctx); err != nil {
@@ -99,9 +120,11 @@ func Test_rkv_configure(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
+			b := atomic.Bool{}
 			rc := &Store{
-				opts: tt.fields.options,
-				cli:  tt.fields.Client,
+				opts:      tt.fields.options,
+				cli:       tt.fields.Client,
+				connected: &b,
 			}
 			err := rc.configure()
 			if (err != nil) != tt.wantErr {

stats.go

@@ -33,20 +33,17 @@ func (r *Store) statsMeter() {
 		ticker := time.NewTicker(meter.DefaultMeterStatsInterval)
 		defer ticker.Stop()
 
-		for {
-			select {
-			case <-ticker.C:
-				if st == nil {
-					return
-				}
-				stats := st.PoolStats()
-				r.opts.Meter.Counter(PoolHitsTotal).Set(uint64(stats.Hits))
-				r.opts.Meter.Counter(PoolMissesTotal).Set(uint64(stats.Misses))
-				r.opts.Meter.Counter(PoolTimeoutTotal).Set(uint64(stats.Timeouts))
-				r.opts.Meter.Counter(PoolConnTotalCurrent).Set(uint64(stats.TotalConns))
-				r.opts.Meter.Counter(PoolConnIdleCurrent).Set(uint64(stats.IdleConns))
-				r.opts.Meter.Counter(PoolConnStaleTotal).Set(uint64(stats.StaleConns))
+		for _ = range ticker.C {
+			if st == nil {
+				return
 			}
+			stats := st.PoolStats()
+			r.opts.Meter.Counter(PoolHitsTotal).Set(uint64(stats.Hits))
+			r.opts.Meter.Counter(PoolMissesTotal).Set(uint64(stats.Misses))
+			r.opts.Meter.Counter(PoolTimeoutTotal).Set(uint64(stats.Timeouts))
+			r.opts.Meter.Counter(PoolConnTotalCurrent).Set(uint64(stats.TotalConns))
+			r.opts.Meter.Counter(PoolConnIdleCurrent).Set(uint64(stats.IdleConns))
+			r.opts.Meter.Counter(PoolConnStaleTotal).Set(uint64(stats.StaleConns))
 		}
 	}()
 }

tracer.go

@@ -73,7 +73,7 @@ func (h *tracingHook) ProcessHook(hook goredis.ProcessHook) goredis.ProcessHook
 		case "cluster slots":
 			break
 		default:
-			_, span := h.tr.Start(ctx, "goredis.process", append(h.opts, tracer.WithSpanLabels("db.statement", cmdString))...)
+			_, span := h.tr.Start(ctx, "sdk.database", append(h.opts, tracer.WithSpanLabels("db.statement", cmdString))...)
 			defer func() {
 				recordError(span, err)
 				span.Finish()
@@ -91,11 +91,11 @@ func (h *tracingHook) ProcessPipelineHook(hook goredis.ProcessPipelineHook) gore
 		_, cmdsString := rediscmd.CmdsString(cmds)
 
 		opts := append(h.opts, tracer.WithSpanLabels(
-			"db.goredis.num_cmd", strconv.Itoa(len(cmds)),
+			"db.database.num_cmd", strconv.Itoa(len(cmds)),
 			"db.statement", cmdsString,
 		))
 
-		_, span := h.tr.Start(ctx, "goredis.process_pipeline", opts...)
+		_, span := h.tr.Start(ctx, "sdk.database", opts...)
 		defer span.Finish()
 
 		err := hook(ctx, cmds)