improve metrics and tracing

Signed-off-by: Vasiliy Tolstov <v.tolstov@unistack.org>

go.mod

@@ -1,14 +1,16 @@
 module go.unistack.org/micro-store-redis/v3
 
-go 1.20
+go 1.21
+
+toolchain go1.22.4
 
 require (
-	github.com/redis/go-redis/v9 v9.2.1
-	go.unistack.org/micro/v3 v3.10.62
+	github.com/redis/go-redis/extra/rediscmd/v9 v9.5.3
+	github.com/redis/go-redis/v9 v9.5.3
+	go.unistack.org/micro/v3 v3.10.80
 )
 
 require (
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f // indirect
-	github.com/patrickmn/go-cache v2.1.0+incompatible // indirect
 )

go.sum

@@ -1,12 +1,14 @@
 github.com/bsm/ginkgo/v2 v2.12.0 h1:Ny8MWAHyOepLGlLKYmXG4IEkioBysk6GpaRTLC8zwWs=
+github.com/bsm/ginkgo/v2 v2.12.0/go.mod h1:SwYbGRRDovPVboqFv0tPTcG1sN61LM1Z4ARdbAV9g4c=
 github.com/bsm/gomega v1.27.10 h1:yeMWxP2pV2fG3FgAODIY8EiRE3dy0aeFYt4l7wh6yKA=
+github.com/bsm/gomega v1.27.10/go.mod h1:JyEr/xRbxbtgWNi8tIEVPUYZ5Dzef52k01W3YH0H+O0=
 github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44=
 github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs=
 github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f h1:lO4WD4F/rVNCu3HqELle0jiPLLBs70cWOduZpkS1E78=
 github.com/dgryski/go-rendezvous v0.0.0-20200823014737-9f7001d12a5f/go.mod h1:cuUVRXasLTGF7a8hSLbxyZXjz+1KgoB3wDUb6vlszIc=
-github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
-github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
-github.com/redis/go-redis/v9 v9.2.1 h1:WlYJg71ODF0dVspZZCpYmoF1+U1Jjk9Rwd7pq6QmlCg=
-github.com/redis/go-redis/v9 v9.2.1/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M=
-go.unistack.org/micro/v3 v3.10.62 h1:PCwLSt3W53UGosH/5qU3kU0iJxK8jlKOm9p4v/Zti5o=
-go.unistack.org/micro/v3 v3.10.62/go.mod h1:erMgt3Bl7vQQ0e9UpQyR5NlLiZ9pKeEJ9+1tfYFaqUg=
+github.com/redis/go-redis/extra/rediscmd/v9 v9.5.3 h1:1/BDligzCa40GTllkDnY3Y5DTHuKCONbB2JcRyIfl20=
+github.com/redis/go-redis/extra/rediscmd/v9 v9.5.3/go.mod h1:3dZmcLn3Qw6FLlWASn1g4y+YO9ycEFUOM+bhBmzLVKQ=
+github.com/redis/go-redis/v9 v9.5.3 h1:fOAp1/uJG+ZtcITgZOfYFmTKPE7n4Vclj1wZFgRciUU=
+github.com/redis/go-redis/v9 v9.5.3/go.mod h1:hdY0cQFCN4fnSYT6TkisLufl/4W5UIXyv0b/CLO2V2M=
+go.unistack.org/micro/v3 v3.10.80 h1:A0zWNoM9MOcMg9gdFFgVkgbT3uSYVIINhuvumX9nP2o=
+go.unistack.org/micro/v3 v3.10.80/go.mod h1:erMgt3Bl7vQQ0e9UpQyR5NlLiZ9pKeEJ9+1tfYFaqUg=

options.go

@@ -1,8 +1,13 @@
 package redis
 
 import (
+	"time"
+
 	"github.com/redis/go-redis/v9"
+	"go.unistack.org/micro/v3/logger"
+	"go.unistack.org/micro/v3/meter"
 	"go.unistack.org/micro/v3/store"
+	"go.unistack.org/micro/v3/tracer"
 )
 
 type configKey struct{}
@@ -16,3 +21,59 @@ type clusterConfigKey struct{}
 func ClusterConfig(c *redis.ClusterOptions) store.Option {
 	return store.SetOption(clusterConfigKey{}, c)
 }
+
+var (
+	// DefaultMeterStatsInterval holds default stats interval
+	DefaultMeterStatsInterval = 5 * time.Second
+	// DefaultMeterMetricPrefix holds default metric prefix
+	DefaultMeterMetricPrefix = "micro_store_"
+)
+
+// Options struct holds wrapper options
+type Options struct {
+	Logger             logger.Logger
+	Meter              meter.Meter
+	Tracer             tracer.Tracer
+	MeterMetricPrefix  string
+	MeterStatsInterval time.Duration
+}
+
+// Option func signature
+type Option func(*Options)
+
+// NewOptions create new Options struct from provided option slice
+func NewOptions(opts ...Option) Options {
+	options := Options{
+		Logger:             logger.DefaultLogger,
+		Meter:              meter.DefaultMeter,
+		Tracer:             tracer.DefaultTracer,
+		MeterStatsInterval: DefaultMeterStatsInterval,
+		MeterMetricPrefix:  DefaultMeterMetricPrefix,
+	}
+
+	for _, o := range opts {
+		o(&options)
+	}
+
+	options.Meter = options.Meter.Clone(
+		meter.MetricPrefix(options.MeterMetricPrefix),
+	)
+
+	options.Logger = options.Logger.Clone(logger.WithCallerSkipCount(1))
+
+	return options
+}
+
+// MetricInterval specifies stats interval for *sql.DB
+func MetricInterval(td time.Duration) Option {
+	return func(o *Options) {
+		o.MeterStatsInterval = td
+	}
+}
+
+// MetricPrefix specifies prefix for each metric
+func MetricPrefix(pref string) Option {
+	return func(o *Options) {
+		o.MeterMetricPrefix = pref
+	}
+}

redis.go

@@ -46,6 +46,7 @@ var (
 type Store struct {
 	opts store.Options
 	cli  *wrappedClient
+	done chan struct{}
 	pool pool.Pool[*strings.Builder]
 }
 
@@ -66,7 +67,12 @@ func (r *Store) Init(opts ...store.Option) error {
 		o(&r.opts)
 	}
 
-	return r.configure()
+	err := r.configure()
+	if err != nil {
+		return err
+	}
+
+	return nil
 }
 
 func (r *Store) Client() *redis.Client {
@@ -84,10 +90,21 @@ func (r *Store) ClusterClient() *redis.ClusterClient {
 }
 
 func (r *Store) Disconnect(ctx context.Context) error {
+	var err error
+	select {
+	case <-r.done:
+		return err
+	default:
 		if r.cli.Client != nil {
-		return r.cli.Client.Close()
+			err = r.cli.Client.Close()
+		} else if r.cli.ClusterClient != nil {
+			err = r.cli.ClusterClient.Close()
 		}
-	return r.cli.ClusterClient.Close()
+		close(r.done)
+		return err
+	}
+
+	return err
 }
 
 func (r *Store) Exists(ctx context.Context, key string, opts ...store.ExistsOption) error {
@@ -108,7 +125,7 @@ func (r *Store) Exists(ctx context.Context, key string, opts ...store.ExistsOpti
 	ctx, sp := r.opts.Tracer.Start(ctx, "cache exists "+rkey)
 	defer sp.Finish()
 
-	r.opts.Meter.Counter(semconv.CacheRequestInflight, "name", options.Name).Inc()
+	r.opts.Meter.Counter(semconv.StoreRequestInflight, "name", options.Name).Inc()
 	ts := time.Now()
 	var err error
 	var val int64
@@ -118,17 +135,17 @@ func (r *Store) Exists(ctx context.Context, key string, opts ...store.ExistsOpti
 		val, err = r.cli.ClusterClient.Exists(ctx, rkey).Result()
 	}
 	te := time.Since(ts)
-	r.opts.Meter.Counter(semconv.CacheRequestInflight, "name", options.Name).Dec()
-	r.opts.Meter.Summary(semconv.CacheRequestLatencyMicroseconds, "name", options.Name).Update(te.Seconds())
-	r.opts.Meter.Histogram(semconv.CacheRequestDurationSeconds, "name", options.Name).Update(te.Seconds())
+	r.opts.Meter.Counter(semconv.StoreRequestInflight, "name", options.Name).Dec()
+	r.opts.Meter.Summary(semconv.StoreRequestLatencyMicroseconds, "name", options.Name).Update(te.Seconds())
+	r.opts.Meter.Histogram(semconv.StoreRequestDurationSeconds, "name", options.Name).Update(te.Seconds())
 	if err == redis.Nil || (err == nil && val == 0) {
-		r.opts.Meter.Counter(semconv.CacheRequestTotal, "name", options.Name, "status", "miss").Inc()
+		r.opts.Meter.Counter(semconv.StoreRequestTotal, "name", options.Name, "status", "miss").Inc()
 		return store.ErrNotFound
 	} else if err == nil {
-		r.opts.Meter.Counter(semconv.CacheRequestTotal, "name", options.Name, "status", "hit").Inc()
+		r.opts.Meter.Counter(semconv.StoreRequestTotal, "name", options.Name, "status", "hit").Inc()
 	} else if err != nil {
 		sp.SetStatus(tracer.SpanStatusError, err.Error())
-		r.opts.Meter.Counter(semconv.CacheRequestTotal, "name", options.Name, "status", "failure").Inc()
+		r.opts.Meter.Counter(semconv.StoreRequestTotal, "name", options.Name, "status", "failure").Inc()
 		return err
 	}
 
@@ -153,7 +170,7 @@ func (r *Store) Read(ctx context.Context, key string, val interface{}, opts ...s
 	ctx, sp := r.opts.Tracer.Start(ctx, "cache read "+rkey)
 	defer sp.Finish()
 
-	r.opts.Meter.Counter(semconv.CacheRequestInflight, "name", options.Name).Inc()
+	r.opts.Meter.Counter(semconv.StoreRequestInflight, "name", options.Name).Inc()
 	ts := time.Now()
 	var buf []byte
 	var err error
@@ -163,17 +180,17 @@ func (r *Store) Read(ctx context.Context, key string, val interface{}, opts ...s
 		buf, err = r.cli.ClusterClient.Get(ctx, rkey).Bytes()
 	}
 	te := time.Since(ts)
-	r.opts.Meter.Counter(semconv.CacheRequestInflight, "name", options.Name).Dec()
-	r.opts.Meter.Summary(semconv.CacheRequestLatencyMicroseconds, "name", options.Name).Update(te.Seconds())
-	r.opts.Meter.Histogram(semconv.CacheRequestDurationSeconds, "name", options.Name).Update(te.Seconds())
+	r.opts.Meter.Counter(semconv.StoreRequestInflight, "name", options.Name).Dec()
+	r.opts.Meter.Summary(semconv.StoreRequestLatencyMicroseconds, "name", options.Name).Update(te.Seconds())
+	r.opts.Meter.Histogram(semconv.StoreRequestDurationSeconds, "name", options.Name).Update(te.Seconds())
 	if err == redis.Nil || (err == nil && buf == nil) {
-		r.opts.Meter.Counter(semconv.CacheRequestTotal, "name", options.Name, "status", "miss").Inc()
+		r.opts.Meter.Counter(semconv.StoreRequestTotal, "name", options.Name, "status", "miss").Inc()
 		return store.ErrNotFound
 	} else if err == nil {
-		r.opts.Meter.Counter(semconv.CacheRequestTotal, "name", options.Name, "status", "hit").Inc()
+		r.opts.Meter.Counter(semconv.StoreRequestTotal, "name", options.Name, "status", "hit").Inc()
 	} else if err != nil {
 		sp.SetStatus(tracer.SpanStatusError, err.Error())
-		r.opts.Meter.Counter(semconv.CacheRequestTotal, "name", options.Name, "status", "failure").Inc()
+		r.opts.Meter.Counter(semconv.StoreRequestTotal, "name", options.Name, "status", "failure").Inc()
 		return err
 	}
 
@@ -214,7 +231,7 @@ func (r *Store) MRead(ctx context.Context, keys []string, vals interface{}, opts
 	ctx, sp := r.opts.Tracer.Start(ctx, fmt.Sprintf("cache mread %v", keys))
 	defer sp.Finish()
 
-	r.opts.Meter.Counter(semconv.CacheRequestInflight, "name", options.Name).Inc()
+	r.opts.Meter.Counter(semconv.StoreRequestInflight, "name", options.Name).Inc()
 	ts := time.Now()
 	var rvals []interface{}
 	var err error
@@ -224,17 +241,17 @@ func (r *Store) MRead(ctx context.Context, keys []string, vals interface{}, opts
 		rvals, err = r.cli.ClusterClient.MGet(ctx, keys...).Result()
 	}
 	te := time.Since(ts)
-	r.opts.Meter.Counter(semconv.CacheRequestInflight, "name", options.Name).Dec()
-	r.opts.Meter.Summary(semconv.CacheRequestLatencyMicroseconds, "name", options.Name).Update(te.Seconds())
-	r.opts.Meter.Histogram(semconv.CacheRequestDurationSeconds, "name", options.Name).Update(te.Seconds())
+	r.opts.Meter.Counter(semconv.StoreRequestInflight, "name", options.Name).Dec()
+	r.opts.Meter.Summary(semconv.StoreRequestLatencyMicroseconds, "name", options.Name).Update(te.Seconds())
+	r.opts.Meter.Histogram(semconv.StoreRequestDurationSeconds, "name", options.Name).Update(te.Seconds())
 	if err == redis.Nil || (len(rvals) == 0) {
-		r.opts.Meter.Counter(semconv.CacheRequestTotal, "name", options.Name, "status", "miss").Inc()
+		r.opts.Meter.Counter(semconv.StoreRequestTotal, "name", options.Name, "status", "miss").Inc()
 		return store.ErrNotFound
 	} else if err == nil {
-		r.opts.Meter.Counter(semconv.CacheRequestTotal, "name", options.Name, "status", "hit").Inc()
+		r.opts.Meter.Counter(semconv.StoreRequestTotal, "name", options.Name, "status", "hit").Inc()
 	} else if err != nil {
 		sp.SetStatus(tracer.SpanStatusError, err.Error())
-		r.opts.Meter.Counter(semconv.CacheRequestTotal, "name", options.Name, "status", "failure").Inc()
+		r.opts.Meter.Counter(semconv.StoreRequestTotal, "name", options.Name, "status", "failure").Inc()
 		return err
 	}
 
@@ -307,7 +324,7 @@ func (r *Store) MDelete(ctx context.Context, keys []string, opts ...store.Delete
 	ctx, sp := r.opts.Tracer.Start(ctx, fmt.Sprintf("cache mdelete %v", keys))
 	defer sp.Finish()
 
-	r.opts.Meter.Counter(semconv.CacheRequestInflight, "name", options.Name).Inc()
+	r.opts.Meter.Counter(semconv.StoreRequestInflight, "name", options.Name).Inc()
 	ts := time.Now()
 	var err error
 	if r.cli.Client != nil {
@@ -316,17 +333,17 @@ func (r *Store) MDelete(ctx context.Context, keys []string, opts ...store.Delete
 		err = r.cli.ClusterClient.Del(ctx, keys...).Err()
 	}
 	te := time.Since(ts)
-	r.opts.Meter.Counter(semconv.CacheRequestInflight, "name", options.Name).Dec()
-	r.opts.Meter.Summary(semconv.CacheRequestLatencyMicroseconds, "name", options.Name).Update(te.Seconds())
-	r.opts.Meter.Histogram(semconv.CacheRequestDurationSeconds, "name", options.Name).Update(te.Seconds())
+	r.opts.Meter.Counter(semconv.StoreRequestInflight, "name", options.Name).Dec()
+	r.opts.Meter.Summary(semconv.StoreRequestLatencyMicroseconds, "name", options.Name).Update(te.Seconds())
+	r.opts.Meter.Histogram(semconv.StoreRequestDurationSeconds, "name", options.Name).Update(te.Seconds())
 	if err == redis.Nil {
-		r.opts.Meter.Counter(semconv.CacheRequestTotal, "name", options.Name, "status", "miss").Inc()
+		r.opts.Meter.Counter(semconv.StoreRequestTotal, "name", options.Name, "status", "miss").Inc()
 		return store.ErrNotFound
 	} else if err == nil {
-		r.opts.Meter.Counter(semconv.CacheRequestTotal, "name", options.Name, "status", "hit").Inc()
+		r.opts.Meter.Counter(semconv.StoreRequestTotal, "name", options.Name, "status", "hit").Inc()
 	} else if err != nil {
 		sp.SetStatus(tracer.SpanStatusError, err.Error())
-		r.opts.Meter.Counter(semconv.CacheRequestTotal, "name", options.Name, "status", "failure").Inc()
+		r.opts.Meter.Counter(semconv.StoreRequestTotal, "name", options.Name, "status", "failure").Inc()
 		return err
 	}
 
@@ -350,7 +367,7 @@ func (r *Store) Delete(ctx context.Context, key string, opts ...store.DeleteOpti
 	ctx, sp := r.opts.Tracer.Start(ctx, fmt.Sprintf("cache delete %v", key))
 	defer sp.Finish()
 
-	r.opts.Meter.Counter(semconv.CacheRequestInflight, "name", options.Name).Inc()
+	r.opts.Meter.Counter(semconv.StoreRequestInflight, "name", options.Name).Inc()
 	ts := time.Now()
 	var err error
 	if r.cli.Client != nil {
@@ -359,17 +376,17 @@ func (r *Store) Delete(ctx context.Context, key string, opts ...store.DeleteOpti
 		err = r.cli.ClusterClient.Del(ctx, r.getKey(r.opts.Namespace, options.Namespace, key)).Err()
 	}
 	te := time.Since(ts)
-	r.opts.Meter.Counter(semconv.CacheRequestInflight, "name", options.Name).Dec()
-	r.opts.Meter.Summary(semconv.CacheRequestLatencyMicroseconds, "name", options.Name).Update(te.Seconds())
-	r.opts.Meter.Histogram(semconv.CacheRequestDurationSeconds, "name", options.Name).Update(te.Seconds())
+	r.opts.Meter.Counter(semconv.StoreRequestInflight, "name", options.Name).Dec()
+	r.opts.Meter.Summary(semconv.StoreRequestLatencyMicroseconds, "name", options.Name).Update(te.Seconds())
+	r.opts.Meter.Histogram(semconv.StoreRequestDurationSeconds, "name", options.Name).Update(te.Seconds())
 	if err == redis.Nil {
-		r.opts.Meter.Counter(semconv.CacheRequestTotal, "name", options.Name, "status", "miss").Inc()
+		r.opts.Meter.Counter(semconv.StoreRequestTotal, "name", options.Name, "status", "miss").Inc()
 		return store.ErrNotFound
 	} else if err == nil {
-		r.opts.Meter.Counter(semconv.CacheRequestTotal, "name", options.Name, "status", "hit").Inc()
+		r.opts.Meter.Counter(semconv.StoreRequestTotal, "name", options.Name, "status", "hit").Inc()
 	} else if err != nil {
 		sp.SetStatus(tracer.SpanStatusError, err.Error())
-		r.opts.Meter.Counter(semconv.CacheRequestTotal, "name", options.Name, "status", "failure").Inc()
+		r.opts.Meter.Counter(semconv.StoreRequestTotal, "name", options.Name, "status", "failure").Inc()
 		return err
 	}
 
@@ -413,7 +430,7 @@ func (r *Store) MWrite(ctx context.Context, keys []string, vals []interface{}, o
 		}
 	}
 
-	r.opts.Meter.Counter(semconv.CacheRequestInflight, "name", options.Name).Inc()
+	r.opts.Meter.Counter(semconv.StoreRequestInflight, "name", options.Name).Inc()
 	ts := time.Now()
 
 	pipeliner := func(pipe redis.Pipeliner) error {
@@ -435,28 +452,28 @@ func (r *Store) MWrite(ctx context.Context, keys []string, vals []interface{}, o
 	}
 
 	te := time.Since(ts)
-	r.opts.Meter.Counter(semconv.CacheRequestInflight, "name", options.Name).Dec()
-	r.opts.Meter.Summary(semconv.CacheRequestLatencyMicroseconds, "name", options.Name).Update(te.Seconds())
-	r.opts.Meter.Histogram(semconv.CacheRequestDurationSeconds, "name", options.Name).Update(te.Seconds())
+	r.opts.Meter.Counter(semconv.StoreRequestInflight, "name", options.Name).Dec()
+	r.opts.Meter.Summary(semconv.StoreRequestLatencyMicroseconds, "name", options.Name).Update(te.Seconds())
+	r.opts.Meter.Histogram(semconv.StoreRequestDurationSeconds, "name", options.Name).Update(te.Seconds())
 	if err == redis.Nil {
-		r.opts.Meter.Counter(semconv.CacheRequestTotal, "name", options.Name, "status", "miss").Inc()
+		r.opts.Meter.Counter(semconv.StoreRequestTotal, "name", options.Name, "status", "miss").Inc()
 		return store.ErrNotFound
 	} else if err == nil {
-		r.opts.Meter.Counter(semconv.CacheRequestTotal, "name", options.Name, "status", "hit").Inc()
+		r.opts.Meter.Counter(semconv.StoreRequestTotal, "name", options.Name, "status", "hit").Inc()
 	} else if err != nil {
 		sp.SetStatus(tracer.SpanStatusError, err.Error())
-		r.opts.Meter.Counter(semconv.CacheRequestTotal, "name", options.Name, "status", "failure").Inc()
+		r.opts.Meter.Counter(semconv.StoreRequestTotal, "name", options.Name, "status", "failure").Inc()
 		return err
 	}
 
 	for _, cmd := range cmds {
 		if err = cmd.Err(); err != nil {
 			if err == redis.Nil {
-				r.opts.Meter.Counter(semconv.CacheRequestTotal, "name", options.Name, "status", "miss").Inc()
+				r.opts.Meter.Counter(semconv.StoreRequestTotal, "name", options.Name, "status", "miss").Inc()
 				return store.ErrNotFound
 			}
 			sp.SetStatus(tracer.SpanStatusError, err.Error())
-			r.opts.Meter.Counter(semconv.CacheRequestTotal, "name", options.Name, "status", "failure").Inc()
+			r.opts.Meter.Counter(semconv.StoreRequestTotal, "name", options.Name, "status", "failure").Inc()
 			return err
 		}
 	}
@@ -497,7 +514,7 @@ func (r *Store) Write(ctx context.Context, key string, val interface{}, opts ...
 		}
 	}
 
-	r.opts.Meter.Counter(semconv.CacheRequestInflight, "name", options.Name).Inc()
+	r.opts.Meter.Counter(semconv.StoreRequestInflight, "name", options.Name).Inc()
 	ts := time.Now()
 	var err error
 	if r.cli.Client != nil {
@@ -506,17 +523,17 @@ func (r *Store) Write(ctx context.Context, key string, val interface{}, opts ...
 		err = r.cli.ClusterClient.Set(ctx, rkey, buf, options.TTL).Err()
 	}
 	te := time.Since(ts)
-	r.opts.Meter.Counter(semconv.CacheRequestInflight, "name", options.Name).Dec()
-	r.opts.Meter.Summary(semconv.CacheRequestLatencyMicroseconds, "name", options.Name).Update(te.Seconds())
-	r.opts.Meter.Histogram(semconv.CacheRequestDurationSeconds, "name", options.Name).Update(te.Seconds())
+	r.opts.Meter.Counter(semconv.StoreRequestInflight, "name", options.Name).Dec()
+	r.opts.Meter.Summary(semconv.StoreRequestLatencyMicroseconds, "name", options.Name).Update(te.Seconds())
+	r.opts.Meter.Histogram(semconv.StoreRequestDurationSeconds, "name", options.Name).Update(te.Seconds())
 	if err == redis.Nil {
-		r.opts.Meter.Counter(semconv.CacheRequestTotal, "name", options.Name, "status", "miss").Inc()
+		r.opts.Meter.Counter(semconv.StoreRequestTotal, "name", options.Name, "status", "miss").Inc()
 		return store.ErrNotFound
 	} else if err == nil {
-		r.opts.Meter.Counter(semconv.CacheRequestTotal, "name", options.Name, "status", "hit").Inc()
+		r.opts.Meter.Counter(semconv.StoreRequestTotal, "name", options.Name, "status", "hit").Inc()
 	} else if err != nil {
 		sp.SetStatus(tracer.SpanStatusError, err.Error())
-		r.opts.Meter.Counter(semconv.CacheRequestTotal, "name", options.Name, "status", "failure").Inc()
+		r.opts.Meter.Counter(semconv.StoreRequestTotal, "name", options.Name, "status", "failure").Inc()
 		return err
 	}
 
@@ -549,7 +566,7 @@ func (r *Store) List(ctx context.Context, opts ...store.ListOption) ([]string, e
 	defer sp.Finish()
 
 	// TODO: add support for prefix/suffix/limit
-	r.opts.Meter.Counter(semconv.CacheRequestInflight, "name", options.Name).Inc()
+	r.opts.Meter.Counter(semconv.StoreRequestInflight, "name", options.Name).Inc()
 	ts := time.Now()
 	var keys []string
 	var err error
@@ -567,17 +584,17 @@ func (r *Store) List(ctx context.Context, opts ...store.ListOption) ([]string, e
 		})
 	}
 	te := time.Since(ts)
-	r.opts.Meter.Counter(semconv.CacheRequestInflight, "name", options.Name).Dec()
-	r.opts.Meter.Summary(semconv.CacheRequestLatencyMicroseconds, "name", options.Name).Update(te.Seconds())
-	r.opts.Meter.Histogram(semconv.CacheRequestDurationSeconds, "name", options.Name).Update(te.Seconds())
+	r.opts.Meter.Counter(semconv.StoreRequestInflight, "name", options.Name).Dec()
+	r.opts.Meter.Summary(semconv.StoreRequestLatencyMicroseconds, "name", options.Name).Update(te.Seconds())
+	r.opts.Meter.Histogram(semconv.StoreRequestDurationSeconds, "name", options.Name).Update(te.Seconds())
 	if err == redis.Nil {
-		r.opts.Meter.Counter(semconv.CacheRequestTotal, "name", options.Name, "status", "miss").Inc()
+		r.opts.Meter.Counter(semconv.StoreRequestTotal, "name", options.Name, "status", "miss").Inc()
 		return nil, store.ErrNotFound
 	} else if err == nil {
-		r.opts.Meter.Counter(semconv.CacheRequestTotal, "name", options.Name, "status", "hit").Inc()
+		r.opts.Meter.Counter(semconv.StoreRequestTotal, "name", options.Name, "status", "hit").Inc()
 	} else if err != nil {
 		sp.SetStatus(tracer.SpanStatusError, err.Error())
-		r.opts.Meter.Counter(semconv.CacheRequestTotal, "name", options.Name, "status", "failure").Inc()
+		r.opts.Meter.Counter(semconv.StoreRequestTotal, "name", options.Name, "status", "failure").Inc()
 		return nil, err
 	}
 
@@ -609,7 +626,7 @@ func (r *Store) String() string {
 }
 
 func NewStore(opts ...store.Option) *Store {
-	return &Store{opts: store.NewOptions(opts...)}
+	return &Store{done: make(chan struct{}), opts: store.NewOptions(opts...)}
 }
 
 func (r *Store) configure() error {
@@ -665,12 +682,19 @@ func (r *Store) configure() error {
 	}
 
 	if redisOptions != nil {
-		r.cli = &wrappedClient{Client: redis.NewClient(redisOptions)}
+		c := redis.NewClient(redisOptions)
+		setTracing(c, r.opts.Tracer)
+		r.cli = &wrappedClient{Client: c}
 	} else if redisClusterOptions != nil {
-		r.cli = &wrappedClient{ClusterClient: redis.NewClusterClient(redisClusterOptions)}
+		c := redis.NewClusterClient(redisClusterOptions)
+		setTracing(c, r.opts.Tracer)
+		r.cli = &wrappedClient{ClusterClient: c}
 	}
 
 	r.pool = pool.NewPool(func() *strings.Builder { return &strings.Builder{} })
+
+	r.statsMeter()
+
 	return nil
 }
 

redis_test.go

@@ -8,8 +8,36 @@ import (
 	"time"
 
 	"go.unistack.org/micro/v3/store"
+	"go.unistack.org/micro/v3/tracer"
 )
 
+func TestKeepTTL(t *testing.T) {
+	ctx := context.Background()
+
+	if tr := os.Getenv("INTEGRATION_TESTS"); len(tr) > 0 {
+		t.Skip()
+	}
+	r := NewStore(store.Addrs(os.Getenv("STORE_NODES")))
+
+	if err := r.Init(); err != nil {
+		t.Fatal(err)
+	}
+	if err := r.Connect(ctx); err != nil {
+		t.Fatal(err)
+	}
+
+	key := "key"
+	err := r.Write(ctx, key, "val1", store.WriteTTL(15*time.Second))
+	if err != nil {
+		t.Fatalf("Write error: %v", err)
+	}
+	time.Sleep(3 * time.Second)
+	err = r.Write(ctx, key, "val2", store.WriteTTL(-1))
+	if err != nil {
+		t.Fatalf("Write error: %v", err)
+	}
+}
+
 func Test_rkv_configure(t *testing.T) {
 	type fields struct {
 		options store.Options
@@ -36,7 +64,7 @@ func Test_rkv_configure(t *testing.T) {
 			},
 		},
 		{
-			name: "legacy Url", fields: fields{options: store.Options{Addrs: []string{"127.0.0.1:6379"}}, Client: nil},
+			name: "legacy Url", fields: fields{options: store.Options{Tracer: tracer.DefaultTracer, Addrs: []string{"127.0.0.1:6379"}}, Client: nil},
 			wantErr: false, want: wantValues{
 				username: "",
 				password: "",
@@ -44,7 +72,7 @@ func Test_rkv_configure(t *testing.T) {
 			},
 		},
 		{
-			name: "New Url", fields: fields{options: store.Options{Addrs: []string{"redis://127.0.0.1:6379"}}, Client: nil},
+			name: "New Url", fields: fields{options: store.Options{Tracer: tracer.DefaultTracer, Addrs: []string{"redis://127.0.0.1:6379"}}, Client: nil},
 			wantErr: false, want: wantValues{
 				username: "",
 				password: "",
@@ -52,7 +80,7 @@ func Test_rkv_configure(t *testing.T) {
 			},
 		},
 		{
-			name: "Url with Pwd", fields: fields{options: store.Options{Addrs: []string{"redis://:password@redis:6379"}}, Client: nil},
+			name: "Url with Pwd", fields: fields{options: store.Options{Tracer: tracer.DefaultTracer, Addrs: []string{"redis://:password@redis:6379"}}, Client: nil},
 			wantErr: false, want: wantValues{
 				username: "",
 				password: "password",
@@ -60,7 +88,7 @@ func Test_rkv_configure(t *testing.T) {
 			},
 		},
 		{
-			name: "Url with username and Pwd", fields: fields{options: store.Options{Addrs: []string{"redis://username:password@redis:6379"}}, Client: nil},
+			name: "Url with username and Pwd", fields: fields{options: store.Options{Tracer: tracer.DefaultTracer, Addrs: []string{"redis://username:password@redis:6379"}}, Client: nil},
 			wantErr: false, want: wantValues{
 				username: "username",
 				password: "password",

stats.go

@@ -0,0 +1,57 @@
+package redis
+
+import (
+	"time"
+
+	"github.com/redis/go-redis/v9"
+)
+
+var (
+	PoolHitsTotal        = "pool_hits_total"
+	PoolMissesTotal      = "pool_misses_total"
+	PoolTimeoutTotal     = "pool_timeout_total"
+	PoolConnTotalCurrent = "pool_conn_total_current"
+	PoolConnIdleCurrent  = "pool_conn_idle_current"
+	PoolConnStaleTotal   = "pool_conn_stale_total"
+
+	meterRequestTotal               = "request_total"
+	meterRequestLatencyMicroseconds = "latency_microseconds"
+	meterRequestDurationSeconds     = "request_duration_seconds"
+)
+
+type Statser interface {
+	PoolStats() *redis.PoolStats
+}
+
+func (r *Store) statsMeter() {
+	var st Statser
+
+	if r.cli.Client != nil {
+		st = r.cli.Client
+	} else if r.cli.ClusterClient != nil {
+		st = r.cli.ClusterClient
+	} else {
+		return
+	}
+
+	go func() {
+		ticker := time.NewTicker(DefaultMeterStatsInterval)
+		defer ticker.Stop()
+
+		for {
+			select {
+			case <-ticker.C:
+				if st == nil {
+					return
+				}
+				stats := st.PoolStats()
+				r.opts.Meter.Counter(PoolHitsTotal).Set(uint64(stats.Hits))
+				r.opts.Meter.Counter(PoolMissesTotal).Set(uint64(stats.Misses))
+				r.opts.Meter.Counter(PoolTimeoutTotal).Set(uint64(stats.Timeouts))
+				r.opts.Meter.Counter(PoolConnTotalCurrent).Set(uint64(stats.TotalConns))
+				r.opts.Meter.Counter(PoolConnIdleCurrent).Set(uint64(stats.IdleConns))
+				r.opts.Meter.Counter(PoolConnStaleTotal).Set(uint64(stats.StaleConns))
+			}
+		}
+	}()
+}

tracer.go

@@ -0,0 +1,142 @@
+package redis
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"strconv"
+
+	"github.com/redis/go-redis/extra/rediscmd/v9"
+	"github.com/redis/go-redis/v9"
+	"go.unistack.org/micro/v3/tracer"
+)
+
+func setTracing(rdb redis.UniversalClient, tr tracer.Tracer, opts ...tracer.SpanOption) {
+	switch rdb := rdb.(type) {
+	case *redis.Client:
+		opt := rdb.Options()
+		connString := formatDBConnString(opt.Network, opt.Addr)
+		opts = addServerAttributes(opts, opt.Addr)
+		rdb.AddHook(newTracingHook(connString, tr, opts...))
+	case *redis.ClusterClient:
+		rdb.AddHook(newTracingHook("", tr, opts...))
+		rdb.OnNewNode(func(rdb *redis.Client) {
+			opt := rdb.Options()
+			opts = addServerAttributes(opts, opt.Addr)
+			connString := formatDBConnString(opt.Network, opt.Addr)
+			rdb.AddHook(newTracingHook(connString, tr, opts...))
+		})
+	case *redis.Ring:
+		rdb.AddHook(newTracingHook("", tr, opts...))
+		rdb.OnNewNode(func(rdb *redis.Client) {
+			opt := rdb.Options()
+			opts = addServerAttributes(opts, opt.Addr)
+			connString := formatDBConnString(opt.Network, opt.Addr)
+			rdb.AddHook(newTracingHook(connString, tr, opts...))
+		})
+	}
+}
+
+type tracingHook struct {
+	tr   tracer.Tracer
+	opts []tracer.SpanOption
+}
+
+var _ redis.Hook = (*tracingHook)(nil)
+
+func newTracingHook(connString string, tr tracer.Tracer, opts ...tracer.SpanOption) *tracingHook {
+	opts = append(opts, tracer.WithSpanKind(tracer.SpanKindClient))
+	if connString != "" {
+		opts = append(opts, tracer.WithSpanLabels("db.connection_string", connString))
+	}
+
+	return &tracingHook{
+		tr:   tr,
+		opts: opts,
+	}
+}
+
+func (h *tracingHook) DialHook(hook redis.DialHook) redis.DialHook {
+	return func(ctx context.Context, network, addr string) (net.Conn, error) {
+		ctx, span := h.tr.Start(ctx, "redis.dial", h.opts...)
+		defer span.Finish()
+
+		conn, err := hook(ctx, network, addr)
+		if err != nil {
+			recordError(span, err)
+			return nil, err
+		}
+		return conn, nil
+	}
+}
+
+func (h *tracingHook) ProcessHook(hook redis.ProcessHook) redis.ProcessHook {
+	return func(ctx context.Context, cmd redis.Cmder) error {
+		cmdString := rediscmd.CmdString(cmd)
+
+		ctx, span := h.tr.Start(ctx, cmd.FullName(), append(h.opts, tracer.WithSpanLabels("db.statement", cmdString))...)
+		defer span.Finish()
+
+		if err := hook(ctx, cmd); err != nil {
+			recordError(span, err)
+			return err
+		}
+		return nil
+	}
+}
+
+func (h *tracingHook) ProcessPipelineHook(
+	hook redis.ProcessPipelineHook,
+) redis.ProcessPipelineHook {
+	return func(ctx context.Context, cmds []redis.Cmder) error {
+		summary, cmdsString := rediscmd.CmdsString(cmds)
+
+		opts := append(h.opts, tracer.WithSpanLabels(
+			"db.redis.num_cmd", strconv.Itoa(len(cmds)),
+			"db.statement", cmdsString,
+		))
+
+		ctx, span := h.tr.Start(ctx, "redis.pipeline "+summary, opts...)
+		defer span.Finish()
+
+		if err := hook(ctx, cmds); err != nil {
+			recordError(span, err)
+			return err
+		}
+		return nil
+	}
+}
+
+func recordError(span tracer.Span, err error) {
+	if err != redis.Nil {
+		span.SetStatus(tracer.SpanStatusError, err.Error())
+	}
+}
+
+func formatDBConnString(network, addr string) string {
+	if network == "tcp" {
+		network = "redis"
+	}
+	return fmt.Sprintf("%s://%s", network, addr)
+}
+
+// Database span attributes semantic conventions recommended server address and port
+// https://opentelemetry.io/docs/specs/semconv/database/database-spans/#connection-level-attributes
+func addServerAttributes(opts []tracer.SpanOption, addr string) []tracer.SpanOption {
+	host, portString, err := net.SplitHostPort(addr)
+	if err != nil {
+		return opts
+	}
+
+	opts = append(opts, tracer.WithSpanLabels("server.address", host))
+
+	// Parse the port string to an integer
+	port, err := strconv.Atoi(portString)
+	if err != nil {
+		return opts
+	}
+
+	opts = append(opts, tracer.WithSpanLabels("server.port", port))
+
+	return opts
+}