add store.Watcher interface (#123)

Signed-off-by: Vasiliy Tolstov <v.tolstov@unistack.org>
Reviewed-on: #123
Co-authored-by: Vasiliy Tolstov <v.tolstov@unistack.org>
Co-committed-by: Vasiliy Tolstov <v.tolstov@unistack.org>

event.go

@@ -0,0 +1,86 @@
+package redis
+
+import (
+	"context"
+	"errors"
+	"net"
+	"time"
+
+	goredis "github.com/redis/go-redis/v9"
+	"go.unistack.org/micro/v3/store"
+)
+
+type eventHook struct {
+	s *Store
+}
+
+var _ goredis.Hook = (*eventHook)(nil)
+
+func newEventHook(s *Store) *eventHook {
+	return &eventHook{s: s}
+}
+
+func (h *eventHook) DialHook(hook goredis.DialHook) goredis.DialHook {
+	return func(ctx context.Context, network, addr string) (net.Conn, error) {
+		conn, err := hook(ctx, network, addr)
+		if err != nil {
+			if !isRedisError(err) {
+				if h.s.connected.CompareAndSwap(1, 0) {
+					h.s.sendEvent(&event{ts: time.Now(), err: err, t: store.EventTypeDisconnect})
+				}
+			} else {
+				h.s.connected.Store(1)
+			}
+		} else {
+			if h.s.connected.CompareAndSwap(0, 1) {
+				h.s.sendEvent(&event{ts: time.Now(), err: err, t: store.EventTypeConnect})
+			}
+		}
+		return conn, err
+	}
+}
+
+func (h *eventHook) ProcessHook(hook goredis.ProcessHook) goredis.ProcessHook {
+	return func(ctx context.Context, cmd goredis.Cmder) error {
+		err := hook(ctx, cmd)
+		if err != nil {
+			if !isRedisError(err) {
+				if h.s.connected.CompareAndSwap(1, 0) {
+					h.s.sendEvent(&event{ts: time.Now(), err: err, t: store.EventTypeDisconnect})
+				}
+			} else {
+				h.s.connected.Store(1)
+			}
+		} else {
+			if h.s.connected.CompareAndSwap(0, 1) {
+				h.s.sendEvent(&event{ts: time.Now(), err: err, t: store.EventTypeConnect})
+			}
+		}
+		return err
+	}
+}
+
+func (h *eventHook) ProcessPipelineHook(hook goredis.ProcessPipelineHook) goredis.ProcessPipelineHook {
+	return func(ctx context.Context, cmds []goredis.Cmder) error {
+		err := hook(ctx, cmds)
+		if err != nil {
+			if !isRedisError(err) {
+				if h.s.connected.CompareAndSwap(1, 0) {
+					h.s.sendEvent(&event{ts: time.Now(), err: err, t: store.EventTypeDisconnect})
+				}
+			} else {
+				h.s.connected.Store(1)
+			}
+		} else {
+			if h.s.connected.CompareAndSwap(0, 1) {
+				h.s.sendEvent(&event{ts: time.Now(), err: err, t: store.EventTypeConnect})
+			}
+		}
+		return err
+	}
+}
+
+func isRedisError(err error) bool {
+	var rerr goredis.Error
+	return errors.As(err, &rerr)
+}

go.mod

@@ -5,9 +5,9 @@ go 1.22
 toolchain go1.22.4
 
 require (
-	github.com/redis/go-redis/extra/rediscmd/v9 v9.6.2
+	github.com/redis/go-redis/extra/rediscmd/v9 v9.7.0
-	github.com/redis/go-redis/v9 v9.6.2
+	github.com/redis/go-redis/v9 v9.7.0
-	go.unistack.org/micro/v3 v3.10.97
+	go.unistack.org/micro/v3 v3.10.108
 )
 
 require (
@@ -15,5 +15,5 @@ require (
 	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
 	github.com/google/go-cmp v0.6.0 // indirect
 	go.unistack.org/micro-proto/v3 v3.4.1 // indirect
-	google.golang.org/protobuf v1.35.1 // indirect
+	google.golang.org/protobuf v1.35.2 // indirect
 )

go.sum

@@ -8,13 +8,13 @@ github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/r
 github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
 github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
 github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
-github.com/redis/go-redis/extra/rediscmd/v9 v9.6.2 h1:oBlErygFka9FAVfowzW7sRUfB7n31YX0aFzfaOflM3w=
+github.com/redis/go-redis/extra/rediscmd/v9 v9.7.0 h1:BIx9TNZH/Jsr4l1i7VVxnV0JPiwYj8qyrHyuL0fGZrk=
-github.com/redis/go-redis/extra/rediscmd/v9 v9.6.2/go.mod h1:jYLUE5tC8UsFnpSclhEjZlFLMMtHH4jTfFMnshNWkoo=
+github.com/redis/go-redis/extra/rediscmd/v9 v9.7.0/go.mod h1:eTg/YQtGYAZD5r3DlGlJptJ45AHA+/G+2NPn30PKzik=
-github.com/redis/go-redis/v9 v9.6.2 h1:w0uvkRbc9KpgD98zcvo5IrVUsn0lXpRMuhNgiHDJzdk=
+github.com/redis/go-redis/v9 v9.7.0 h1:HhLSs+B6O021gwzl+locl0zEDnyNkxMtf/Z3NNBMa9E=
-github.com/redis/go-redis/v9 v9.6.2/go.mod h1:0C0c6ycQsdpVNQpxb1njEQIqkx5UcsM8FJCQLgE9+RA=
+github.com/redis/go-redis/v9 v9.7.0/go.mod h1:f6zhXITC7JUJIlPEiBOTXxJgPLdZcA93GewI7inzyWw=
 go.unistack.org/micro-proto/v3 v3.4.1 h1:UTjLSRz2YZuaHk9iSlVqqsA50JQNAEK2ZFboGqtEa9Q=
 go.unistack.org/micro-proto/v3 v3.4.1/go.mod h1:okx/cnOhzuCX0ggl/vToatbCupi0O44diiiLLsZ93Zo=
-go.unistack.org/micro/v3 v3.10.97 h1:8l7fv+i06/PjPrBBhRC/ZQkWGIOuHPg3jJN0vktYE78=
+go.unistack.org/micro/v3 v3.10.108 h1:3L7SkilMVLtH8y3pQIPtr3jjQYrf0AMv1oAkoL3nFkE=
-go.unistack.org/micro/v3 v3.10.97/go.mod h1:YzMldzHN9Ei+zy5t/Psu7RUWDZwUfrNYiStSQtTz90g=
+go.unistack.org/micro/v3 v3.10.108/go.mod h1:YzMldzHN9Ei+zy5t/Psu7RUWDZwUfrNYiStSQtTz90g=
-google.golang.org/protobuf v1.35.1 h1:m3LfL6/Ca+fqnjnlqQXNpFPABW1UD7mjh8KO2mKFytA=
+google.golang.org/protobuf v1.35.2 h1:8Ar7bF+apOIoThw1EdZl0p1oWvMqTHmpA2fRTyZO8io=
-google.golang.org/protobuf v1.35.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE=
+google.golang.org/protobuf v1.35.2/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE=

redis.go

@@ -5,17 +5,21 @@ import (
 	"errors"
 	"reflect"
 	"strings"
+	"sync"
+	"sync/atomic"
 	"time"
 
 	goredis "github.com/redis/go-redis/v9"
 	"go.unistack.org/micro/v3/semconv"
 	"go.unistack.org/micro/v3/store"
+	"go.unistack.org/micro/v3/util/id"
 	pool "go.unistack.org/micro/v3/util/xpool"
 )
 
 var (
-	DefaultPathSeparator = "/"
+	_                       store.Store = (*Store)(nil)
-
+	_                       store.Event = (*event)(nil)
+	sendEventTime                       = 10 * time.Millisecond
 	DefaultUniversalOptions             = &goredis.UniversalOptions{
 		Username:        "",
 		Password:        "", // no password set
@@ -55,19 +59,30 @@ var (
 )
 
 type Store struct {
-	opts store.Options
 	cli       goredis.UniversalClient
-	done chan struct{}
 	pool      *pool.StringsPool
+	connected *atomic.Uint32
+	opts      store.Options
+	watchers  map[string]*watcher
+	mu        sync.RWMutex
 }
 
 func (r *Store) Connect(ctx context.Context) error {
+	if r.connected.Load() == 1 {
+		return nil
+	}
 	if r.cli == nil {
 		return store.ErrNotConnected
 	}
-	err := r.cli.Ping(ctx).Err()
+	if r.opts.LazyConnect {
+		return nil
+	}
+	if err := r.cli.Ping(ctx).Err(); err != nil {
 		setSpanError(ctx, err)
 		return err
+	}
+	r.connected.Store(1)
+	return nil
 }
 
 func (r *Store) Init(opts ...store.Option) error {
@@ -102,17 +117,18 @@ func (r *Store) ClusterClient() *goredis.ClusterClient {
 }
 
 func (r *Store) Disconnect(ctx context.Context) error {
-	var err error
+	if r.connected.Load() == 0 {
-	select {
+		return nil
-	case <-r.done:
+	}
-		return err
+
-	default:
 	if r.cli != nil {
-			err = r.cli.Close()
+		if err := r.cli.Close(); err != nil {
-		}
-		close(r.done)
 			return err
 		}
+	}
+
+	r.connected.Store(1)
+	return nil
 }
 
 func (r *Store) Exists(ctx context.Context, key string, opts ...store.ExistsOption) error {
@@ -149,7 +165,7 @@ func (r *Store) Exists(ctx context.Context, key string, opts ...store.ExistsOpti
 		return store.ErrNotFound
 	} else if err == nil {
 		r.opts.Meter.Counter(semconv.StoreRequestTotal, append(labels, "status", "hit")...).Inc()
-	} else if err != nil {
+	} else {
 		r.opts.Meter.Counter(semconv.StoreRequestTotal, append(labels, "status", "failure")...).Inc()
 		return err
 	}
@@ -311,6 +327,8 @@ func (r *Store) MRead(ctx context.Context, keys []string, vals interface{}, opts
 
 func (r *Store) MDelete(ctx context.Context, keys []string, opts ...store.DeleteOption) error {
 	options := store.NewDeleteOptions(opts...)
+	labels := make([]string, 0, 6)
+	labels = append(labels, "name", options.Name, "statement", "delete")
 
 	timeout := r.opts.Timeout
 	if options.Timeout > 0 {
@@ -335,7 +353,7 @@ func (r *Store) MDelete(ctx context.Context, keys []string, opts ...store.Delete
 		}
 	}
 
-	r.opts.Meter.Counter(semconv.StoreRequestInflight, "name", options.Name).Inc()
+	r.opts.Meter.Counter(semconv.StoreRequestInflight, labels...).Inc()
 	ts := time.Now()
 	var err error
 	if r.opts.Namespace != "" || options.Namespace != "" {
@@ -348,16 +366,16 @@ func (r *Store) MDelete(ctx context.Context, keys []string, opts ...store.Delete
 	}
 	setSpanError(ctx, err)
 	te := time.Since(ts)
-	r.opts.Meter.Counter(semconv.StoreRequestInflight, "name", options.Name).Dec()
+	r.opts.Meter.Counter(semconv.StoreRequestInflight, labels...).Dec()
-	r.opts.Meter.Summary(semconv.StoreRequestLatencyMicroseconds, "name", options.Name).Update(te.Seconds())
+	r.opts.Meter.Summary(semconv.StoreRequestLatencyMicroseconds, labels...).Update(te.Seconds())
-	r.opts.Meter.Histogram(semconv.StoreRequestDurationSeconds, "name", options.Name).Update(te.Seconds())
+	r.opts.Meter.Histogram(semconv.StoreRequestDurationSeconds, labels...).Update(te.Seconds())
 	if err == goredis.Nil {
-		r.opts.Meter.Counter(semconv.StoreRequestTotal, "name", options.Name, "status", "miss").Inc()
+		r.opts.Meter.Counter(semconv.StoreRequestTotal, append(labels, "status", "miss")...).Inc()
 		return store.ErrNotFound
 	} else if err == nil {
-		r.opts.Meter.Counter(semconv.StoreRequestTotal, "name", options.Name, "status", "hit").Inc()
+		r.opts.Meter.Counter(semconv.StoreRequestTotal, append(labels, "status", "hit")...).Inc()
 	} else if err != nil {
-		r.opts.Meter.Counter(semconv.StoreRequestTotal, "name", options.Name, "status", "failure").Inc()
+		r.opts.Meter.Counter(semconv.StoreRequestTotal, append(labels, "status", "failure")...).Inc()
 		return err
 	}
 
@@ -407,6 +425,8 @@ func (r *Store) Delete(ctx context.Context, key string, opts ...store.DeleteOpti
 
 func (r *Store) MWrite(ctx context.Context, keys []string, vals []interface{}, opts ...store.WriteOption) error {
 	options := store.NewWriteOptions(opts...)
+	labels := make([]string, 0, 6)
+	labels = append(labels, "name", options.Name, "statement", "write")
 
 	timeout := r.opts.Timeout
 	if options.Timeout > 0 {
@@ -440,7 +460,7 @@ func (r *Store) MWrite(ctx context.Context, keys []string, vals []interface{}, o
 		}
 	}
 
-	r.opts.Meter.Counter(semconv.StoreRequestInflight, "name", options.Name).Inc()
+	r.opts.Meter.Counter(semconv.StoreRequestInflight, labels...).Inc()
 
 	pipeliner := func(pipe goredis.Pipeliner) error {
 		for idx := 0; idx < len(kvs); idx += 2 {
@@ -460,27 +480,27 @@ func (r *Store) MWrite(ctx context.Context, keys []string, vals []interface{}, o
 	te := time.Since(ts)
 	setSpanError(ctx, err)
 
-	r.opts.Meter.Counter(semconv.StoreRequestInflight, "name", options.Name).Dec()
+	r.opts.Meter.Counter(semconv.StoreRequestInflight, labels...).Dec()
-	r.opts.Meter.Summary(semconv.StoreRequestLatencyMicroseconds, "name", options.Name).Update(te.Seconds())
+	r.opts.Meter.Summary(semconv.StoreRequestLatencyMicroseconds, labels...).Update(te.Seconds())
-	r.opts.Meter.Histogram(semconv.StoreRequestDurationSeconds, "name", options.Name).Update(te.Seconds())
+	r.opts.Meter.Histogram(semconv.StoreRequestDurationSeconds, labels...).Update(te.Seconds())
 	if err == goredis.Nil {
-		r.opts.Meter.Counter(semconv.StoreRequestTotal, "name", options.Name, "status", "miss").Inc()
+		r.opts.Meter.Counter(semconv.StoreRequestTotal, append(labels, "status", "miss")...).Inc()
 		return store.ErrNotFound
 	} else if err == nil {
-		r.opts.Meter.Counter(semconv.StoreRequestTotal, "name", options.Name, "status", "hit").Inc()
+		r.opts.Meter.Counter(semconv.StoreRequestTotal, append(labels, "status", "hit")...).Inc()
 	} else if err != nil {
-		r.opts.Meter.Counter(semconv.StoreRequestTotal, "name", options.Name, "status", "failure").Inc()
+		r.opts.Meter.Counter(semconv.StoreRequestTotal, append(labels, "status", "failure")...).Inc()
 		return err
 	}
 
 	for _, cmd := range cmds {
 		if err = cmd.Err(); err != nil {
 			if err == goredis.Nil {
-				r.opts.Meter.Counter(semconv.StoreRequestTotal, "name", options.Name, "status", "miss").Inc()
+				r.opts.Meter.Counter(semconv.StoreRequestTotal, append(labels, "status", "miss")...).Inc()
 				return store.ErrNotFound
 			}
 			setSpanError(ctx, err)
-			r.opts.Meter.Counter(semconv.StoreRequestTotal, "name", options.Name, "status", "failure").Inc()
+			r.opts.Meter.Counter(semconv.StoreRequestTotal, append(labels, "status", "failure")...).Inc()
 			return err
 		}
 	}
@@ -537,7 +557,7 @@ func (r *Store) Write(ctx context.Context, key string, val interface{}, opts ...
 		return store.ErrNotFound
 	} else if err == nil {
 		r.opts.Meter.Counter(semconv.StoreRequestTotal, append(labels, "status", "hit")...).Inc()
-	} else if err != nil {
+	} else {
 		r.opts.Meter.Counter(semconv.StoreRequestTotal, append(labels, "status", "failure")...).Inc()
 		return err
 	}
@@ -550,6 +570,9 @@ func (r *Store) List(ctx context.Context, opts ...store.ListOption) ([]string, e
 	defer r.pool.Put(b)
 
 	options := store.NewListOptions(opts...)
+	labels := make([]string, 0, 6)
+	labels = append(labels, "name", options.Name, "statement", "list")
+
 	if len(options.Namespace) == 0 {
 		options.Namespace = r.opts.Namespace
 	}
@@ -571,7 +594,7 @@ func (r *Store) List(ctx context.Context, opts ...store.ListOption) ([]string, e
 	}
 
 	// TODO: add support for prefix/suffix/limit
-	r.opts.Meter.Counter(semconv.StoreRequestInflight, "name", options.Name).Inc()
+	r.opts.Meter.Counter(semconv.StoreRequestInflight, labels...).Inc()
 	ts := time.Now()
 	var keys []string
 	var err error
@@ -591,16 +614,16 @@ func (r *Store) List(ctx context.Context, opts ...store.ListOption) ([]string, e
 	te := time.Since(ts)
 	setSpanError(ctx, err)
 
-	r.opts.Meter.Counter(semconv.StoreRequestInflight, "name", options.Name).Dec()
+	r.opts.Meter.Counter(semconv.StoreRequestInflight, labels...).Dec()
-	r.opts.Meter.Summary(semconv.StoreRequestLatencyMicroseconds, "name", options.Name).Update(te.Seconds())
+	r.opts.Meter.Summary(semconv.StoreRequestLatencyMicroseconds, labels...).Update(te.Seconds())
-	r.opts.Meter.Histogram(semconv.StoreRequestDurationSeconds, "name", options.Name).Update(te.Seconds())
+	r.opts.Meter.Histogram(semconv.StoreRequestDurationSeconds, labels...).Update(te.Seconds())
 	if err == goredis.Nil {
-		r.opts.Meter.Counter(semconv.StoreRequestTotal, "name", options.Name, "status", "miss").Inc()
+		r.opts.Meter.Counter(semconv.StoreRequestTotal, append(labels, "status", "miss")...).Inc()
 		return nil, store.ErrNotFound
 	} else if err == nil {
-		r.opts.Meter.Counter(semconv.StoreRequestTotal, "name", options.Name, "status", "hit").Inc()
+		r.opts.Meter.Counter(semconv.StoreRequestTotal, append(labels, "status", "git")...).Inc()
 	} else if err != nil {
-		r.opts.Meter.Counter(semconv.StoreRequestTotal, "name", options.Name, "status", "failure").Inc()
+		r.opts.Meter.Counter(semconv.StoreRequestTotal, append(labels, "status", "failure")...).Inc()
 		return nil, err
 	}
 
@@ -632,7 +655,12 @@ func (r *Store) String() string {
 }
 
 func NewStore(opts ...store.Option) *Store {
-	return &Store{done: make(chan struct{}), opts: store.NewOptions(opts...)}
+	b := atomic.Uint32{}
+	return &Store{
+		opts:      store.NewOptions(opts...),
+		connected: &b,
+		watchers:  make(map[string]*watcher),
+	}
 }
 
 func (r *Store) configure() error {
@@ -723,6 +751,7 @@ func (r *Store) configure() error {
 
 	r.cli = goredis.NewUniversalClient(universalOptions)
 	setTracing(r.cli, r.opts.Tracer)
+	r.cli.AddHook(newEventHook(r))
 
 	r.pool = pool.NewStringsPool(50)
 
@@ -737,8 +766,99 @@ func (r *Store) getKey(b *strings.Builder, mainNamespace string, opNamespace str
 	}
 	if opNamespace != "" {
 		b.WriteString(opNamespace)
-		b.WriteString(DefaultPathSeparator)
+		b.WriteString(r.opts.Separator)
 	}
 	b.WriteString(key)
 	return b.String()
 }
+
+func (r *Store) Watch(ctx context.Context, opts ...store.WatchOption) (store.Watcher, error) {
+	id, err := id.New()
+	if err != nil {
+		return nil, err
+	}
+	wo, err := store.NewWatchOptions(opts...)
+	if err != nil {
+		return nil, err
+	}
+	// construct the watcher
+	w := &watcher{
+		exit: make(chan bool),
+		ch:   make(chan store.Event),
+		id:   id,
+		opts: wo,
+	}
+
+	r.mu.Lock()
+	r.watchers[w.id] = w
+	r.mu.Unlock()
+
+	return w, nil
+}
+
+func (r *Store) sendEvent(e store.Event) {
+	r.mu.RLock()
+	watchers := make([]*watcher, 0, len(r.watchers))
+	for _, w := range r.watchers {
+		watchers = append(watchers, w)
+	}
+	r.mu.RUnlock()
+	for _, w := range watchers {
+		select {
+		case <-w.exit:
+			r.mu.Lock()
+			delete(r.watchers, w.id)
+			r.mu.Unlock()
+		default:
+			select {
+			case w.ch <- e:
+			case <-time.After(sendEventTime):
+			}
+		}
+	}
+}
+
+type watcher struct {
+	ch   chan store.Event
+	exit chan bool
+	opts store.WatchOptions
+	id   string
+}
+
+func (w *watcher) Next() (store.Event, error) {
+	for {
+		select {
+		case e := <-w.ch:
+			return e, nil
+		case <-w.exit:
+			return nil, store.ErrWatcherStopped
+		}
+	}
+}
+
+func (w *watcher) Stop() {
+	select {
+	case <-w.exit:
+		return
+	default:
+		close(w.exit)
+	}
+}
+
+type event struct {
+	ts  time.Time
+	t   store.EventType
+	err error
+}
+
+func (e *event) Error() error {
+	return e.err
+}
+
+func (e *event) Timestamp() time.Time {
+	return e.ts
+}
+
+func (e *event) Type() store.EventType {
+	return e.t
+}

redis_test.go

@@ -4,6 +4,7 @@ import (
 	"bytes"
 	"context"
 	"os"
+	"sync/atomic"
 	"testing"
 	"time"
 
@@ -40,7 +41,7 @@ func TestKeepTTL(t *testing.T) {
 	}
 	r := NewStore(store.Addrs(os.Getenv("STORE_NODES")))
 
-	if err := r.Init(); err != nil {
+	if err := r.Init(store.LazyConnect(true)); err != nil {
 		t.Fatal(err)
 	}
 	if err := r.Connect(ctx); err != nil {
@@ -119,9 +120,11 @@ func Test_rkv_configure(t *testing.T) {
 	}
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
+			b := atomic.Bool{}
 			rc := &Store{
 				opts:      tt.fields.options,
 				cli:       tt.fields.Client,
+				connected: &b,
 			}
 			err := rc.configure()
 			if (err != nil) != tt.wantErr {