From c9061860113fde5f1d3cf8b34c256c98ac7881af Mon Sep 17 00:00:00 2001 From: Vasiliy Tolstov Date: Sat, 6 Apr 2024 22:03:56 +0300 Subject: [PATCH 01/59] add semconv package Signed-off-by: Vasiliy Tolstov --- semconv/broker.go | 20 ++++++++++++++++++++ semconv/client.go | 12 ++++++++++++ semconv/server.go | 12 ++++++++++++ 3 files changed, 44 insertions(+) create mode 100644 semconv/broker.go create mode 100644 semconv/client.go create mode 100644 semconv/server.go diff --git a/semconv/broker.go b/semconv/broker.go new file mode 100644 index 00000000..9b61b523 --- /dev/null +++ b/semconv/broker.go @@ -0,0 +1,20 @@ +package semconv + +var ( + // PublishMessageDurationSeconds specifies meter metric name + PublishMessageDurationSeconds = "publish_message_duration_seconds" + // PublishMessageLatencyMicroseconds specifies meter metric name + PublishMessageLatencyMicroseconds = "publish_message_latency_microseconds" + // PublishMessageTotal specifies meter metric name + PublishMessageTotal = "publish_message_total" + // PublishMessageInflight specifies meter metric name + PublishMessageInflight = "publish_message_inflight" + // SubscribeMessageDurationSeconds specifies meter metric name + SubscribeMessageDurationSeconds = "subscribe_message_duration_seconds" + // SubscribeMessageLatencyMicroseconds specifies meter metric name + SubscribeMessageLatencyMicroseconds = "subscribe_message_latency_microseconds" + // SubscribeMessageTotal specifies meter metric name + SubscribeMessageTotal = "subscribe_message_total" + // SubscribeMessageInflight specifies meter metric name + SubscribeMessageInflight = "subscribe_message_inflight" +) diff --git a/semconv/client.go b/semconv/client.go new file mode 100644 index 00000000..96671471 --- /dev/null +++ b/semconv/client.go @@ -0,0 +1,12 @@ +package semconv + +var ( + // ClientRequestDurationSeconds specifies meter metric name + ClientRequestDurationSeconds = "client_request_duration_seconds" + // ClientRequestLatencyMicroseconds specifies meter metric name + ClientRequestLatencyMicroseconds = "client_request_latency_microseconds" + // ClientRequestTotal specifies meter metric name + ClientRequestTotal = "client_request_total" + // ClientRequestInflight specifies meter metric name + ClientRequestInflight = "client_request_inflight" +) diff --git a/semconv/server.go b/semconv/server.go new file mode 100644 index 00000000..fedb7074 --- /dev/null +++ b/semconv/server.go @@ -0,0 +1,12 @@ +package semconv + +var ( + // ServerRequestDurationSeconds specifies meter metric name + ServerRequestDurationSeconds = "server_request_duration_seconds" + // ServerRequestLatencyMicroseconds specifies meter metric name + ServerRequestLatencyMicroseconds = "server_request_latency_microseconds" + // ServerRequestTotal specifies meter metric name + ServerRequestTotal = "server_request_total" + // ServerRequestInflight specifies meter metric name + ServerRequestInflight = "server_request_inflight" +) From 67ba7b37538631f517b527625ed03ad5a3f3ead3 Mon Sep 17 00:00:00 2001 From: Vasiliy Tolstov Date: Sat, 6 Apr 2024 23:28:01 +0300 Subject: [PATCH 02/59] cluster: initial import Signed-off-by: Vasiliy Tolstov --- cluster/cluster.go | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 cluster/cluster.go diff --git a/cluster/cluster.go b/cluster/cluster.go new file mode 100644 index 00000000..26549e62 --- /dev/null +++ b/cluster/cluster.go @@ -0,0 +1,41 @@ +package cluster + +import ( + "context" + + "go.unistack.org/micro/v3/metadata" +) + +// Message sent to member in cluster +type Message interface { + // Header returns message headers + Header() metadata.Metadata + // Body returns broker message may be []byte slice or some go struct or interface + Body() interface{} +} + +type Node interface { + // Name returns node name + Name() string + // Address returns node address + Address() string + // Metadata returns node metadata + Metadata() metadata.Metadata +} + +// Cluster interface used for cluster communication across nodes +type Cluster interface { + // Join is used to take an existing members and performing state sync + Join(ctx context.Context, addr ...string) error + // Leave broadcast a leave message and stop listeners + Leave(ctx context.Context) error + // Ping is used to probe live status of the node + Ping(ctx context.Context, node Node, payload []byte) error + // Members returns the cluster members + Members() ([]Node, error) + // Broadcast send message for all members in cluster, if filter is not nil, nodes may be filtered + // by key/value pairs + Broadcast(ctx context.Context, msg Message, filter ...string) error + // Unicast send message to single member in cluster + Unicast(ctx context.Context, node Node, msg Message) error +} From c2808679c33c320f9b241bcb0e88a7565e50ef74 Mon Sep 17 00:00:00 2001 From: devstigneev Date: Sun, 7 Apr 2024 20:53:01 +0300 Subject: [PATCH 03/59] add options in broker --- .gitignore | 3 +++ broker/broker.go | 3 +++ broker/options.go | 18 ++++++++++++------ 3 files changed, 18 insertions(+), 6 deletions(-) diff --git a/.gitignore b/.gitignore index 500d68ca..c2fff381 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,8 @@ # Develop tools /.vscode/ /.idea/ +.idea +.vscode # Binaries for programs and plugins *.exe @@ -13,6 +15,7 @@ _obj _test _build +.DS_Store # Architecture specific extensions/prefixes *.[568vq] diff --git a/broker/broker.go b/broker/broker.go index 11e029ed..80585c4c 100644 --- a/broker/broker.go +++ b/broker/broker.go @@ -4,6 +4,7 @@ package broker // import "go.unistack.org/micro/v3/broker" import ( "context" "errors" + "time" "go.unistack.org/micro/v3/codec" "go.unistack.org/micro/v3/metadata" @@ -17,6 +18,8 @@ var ( ErrNotConnected = errors.New("broker not connected") // ErrDisconnected returns when broker disconnected ErrDisconnected = errors.New("broker disconnected") + // DefaultGracefulTimeout + DefaultGracefulTimeout = 5 * time.Second ) // Broker is an interface used for asynchronous messaging. diff --git a/broker/options.go b/broker/options.go index 0e4a8c75..a27c445a 100644 --- a/broker/options.go +++ b/broker/options.go @@ -3,6 +3,7 @@ package broker import ( "context" "crypto/tls" + "sync" "time" "go.unistack.org/micro/v3/codec" @@ -36,17 +37,22 @@ type Options struct { Name string // Addrs holds the broker address Addrs []string + + Wait *sync.WaitGroup + + GracefulTimeout time.Duration } // NewOptions create new Options func NewOptions(opts ...Option) Options { options := Options{ - Register: register.DefaultRegister, - Logger: logger.DefaultLogger, - Context: context.Background(), - Meter: meter.DefaultMeter, - Codec: codec.DefaultCodec, - Tracer: tracer.DefaultTracer, + Register: register.DefaultRegister, + Logger: logger.DefaultLogger, + Context: context.Background(), + Meter: meter.DefaultMeter, + Codec: codec.DefaultCodec, + Tracer: tracer.DefaultTracer, + GracefulTimeout: DefaultGracefulTimeout, } for _, o := range opts { o(&options) From 8254456c8becf8d7e57ad5b8b5bcd368ed54f812 Mon Sep 17 00:00:00 2001 From: devstigneev Date: Sun, 7 Apr 2024 21:16:50 +0300 Subject: [PATCH 04/59] rename path to sync --- broker/options.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/broker/options.go b/broker/options.go index a27c445a..c5e91263 100644 --- a/broker/options.go +++ b/broker/options.go @@ -3,13 +3,13 @@ package broker import ( "context" "crypto/tls" - "sync" "time" "go.unistack.org/micro/v3/codec" "go.unistack.org/micro/v3/logger" "go.unistack.org/micro/v3/meter" "go.unistack.org/micro/v3/register" + "go.unistack.org/micro/v3/sync" "go.unistack.org/micro/v3/tracer" ) From c020d90cb4ab06e152820e72ed89f4a827cedcfe Mon Sep 17 00:00:00 2001 From: Vasiliy Tolstov Date: Sat, 13 Apr 2024 02:06:51 +0300 Subject: [PATCH 05/59] semconv: add broker group lag Signed-off-by: Vasiliy Tolstov --- semconv/broker.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/semconv/broker.go b/semconv/broker.go index 9b61b523..8c51a8f2 100644 --- a/semconv/broker.go +++ b/semconv/broker.go @@ -17,4 +17,6 @@ var ( SubscribeMessageTotal = "subscribe_message_total" // SubscribeMessageInflight specifies meter metric name SubscribeMessageInflight = "subscribe_message_inflight" + // BrokerGroupLag specifies broker lag + BrokerGroupLag = "broker_lag" ) From b5f8316b57e15f9e438c78481ec790e9cb252a6f Mon Sep 17 00:00:00 2001 From: Vasiliy Tolstov Date: Sat, 13 Apr 2024 02:38:58 +0300 Subject: [PATCH 06/59] semconv: fix broker group lag metric name Signed-off-by: Vasiliy Tolstov --- semconv/broker.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/semconv/broker.go b/semconv/broker.go index 8c51a8f2..4c949225 100644 --- a/semconv/broker.go +++ b/semconv/broker.go @@ -18,5 +18,5 @@ var ( // SubscribeMessageInflight specifies meter metric name SubscribeMessageInflight = "subscribe_message_inflight" // BrokerGroupLag specifies broker lag - BrokerGroupLag = "broker_lag" + BrokerGroupLag = "broker_group_lag" ) From c320d8e5186c27976d1ff530f7034d4848c44e31 Mon Sep 17 00:00:00 2001 From: Vasiliy Tolstov Date: Sat, 13 Apr 2024 12:52:06 +0300 Subject: [PATCH 07/59] store/options: extend options to holds name and timeout Signed-off-by: Vasiliy Tolstov --- store/options.go | 87 ++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 85 insertions(+), 2 deletions(-) diff --git a/store/options.go b/store/options.go index 586969c3..6bbee1a4 100644 --- a/store/options.go +++ b/store/options.go @@ -144,6 +144,10 @@ type ReadOptions struct { Context context.Context // Namespace holds namespace Namespace string + // Name holds mnemonic name + Name string + // Timeout specifies max timeout for operation + Timeout time.Duration } // NewReadOptions fills ReadOptions struct with opts slice @@ -158,6 +162,20 @@ func NewReadOptions(opts ...ReadOption) ReadOptions { // ReadOption sets values in ReadOptions type ReadOption func(r *ReadOptions) +// ReadTimeout pass timeout to ReadOptions +func ReadTimeout(td time.Duration) ReadOption { + return func(o *ReadOptions) { + o.Timeout = td + } +} + +// ReadName pass name to ReadOptions +func ReadName(name string) ReadOption { + return func(o *ReadOptions) { + o.Name = name + } +} + // ReadContext pass context.Context to ReadOptions func ReadContext(ctx context.Context) ReadOption { return func(o *ReadOptions) { @@ -180,6 +198,10 @@ type WriteOptions struct { Metadata metadata.Metadata // Namespace holds namespace Namespace string + // Name holds mnemonic name + Name string + // Timeout specifies max timeout for operation + Timeout time.Duration // TTL specifies key TTL TTL time.Duration } @@ -224,12 +246,30 @@ func WriteNamespace(ns string) WriteOption { } } +// WriteName pass name to WriteOptions +func WriteName(name string) WriteOption { + return func(o *WriteOptions) { + o.Name = name + } +} + +// WriteTimeout pass timeout to WriteOptions +func WriteTimeout(td time.Duration) WriteOption { + return func(o *WriteOptions) { + o.Timeout = td + } +} + // DeleteOptions configures an individual Delete operation type DeleteOptions struct { // Context holds external options Context context.Context // Namespace holds namespace Namespace string + // Name holds mnemonic name + Name string + // Timeout specifies max timeout for operation + Timeout time.Duration } // NewDeleteOptions fills DeleteOptions struct with opts slice @@ -258,14 +298,32 @@ func DeleteNamespace(ns string) DeleteOption { } } +// DeleteName pass name to DeleteOptions +func DeleteName(name string) DeleteOption { + return func(o *DeleteOptions) { + o.Name = name + } +} + +// DeleteTimeout pass timeout to DeleteOptions +func DeleteTimeout(td time.Duration) DeleteOption { + return func(o *DeleteOptions) { + o.Timeout = td + } +} + // ListOptions configures an individual List operation type ListOptions struct { Context context.Context Prefix string Suffix string Namespace string - Limit uint - Offset uint + // Name holds mnemonic name + Name string + Limit uint + Offset uint + // Timeout specifies max timeout for operation + Timeout time.Duration } // NewListOptions fills ListOptions struct with opts slice @@ -322,12 +380,23 @@ func ListNamespace(ns string) ListOption { } } +// ListTimeout pass timeout to ListOptions +func ListTimeout(td time.Duration) ListOption { + return func(o *ListOptions) { + o.Timeout = td + } +} + // ExistsOptions holds options for Exists method type ExistsOptions struct { // Context holds external options Context context.Context // Namespace contains namespace Namespace string + // Name holds mnemonic name + Name string + // Timeout specifies max timeout for operation + Timeout time.Duration } // ExistsOption specifies Exists call options @@ -358,6 +427,20 @@ func ExistsNamespace(ns string) ExistsOption { } } +// ExistsName pass name to exist options +func ExistsName(name string) ExistsOption { + return func(o *ExistsOptions) { + o.Name = name + } +} + +// ExistsTimeout timeout to ListOptions +func ExistsTimeout(td time.Duration) ExistsOption { + return func(o *ExistsOptions) { + o.Timeout = td + } +} + /* // WrapStore adds a store Wrapper to a list of options passed into the store func WrapStore(w Wrapper) Option { From 1bcf71c189dffc82b9253e16f6ca1f993c04c912 Mon Sep 17 00:00:00 2001 From: Vasiliy Tolstov Date: Sun, 14 Apr 2024 00:16:55 +0300 Subject: [PATCH 08/59] util/xpool: package pool Signed-off-by: Vasiliy Tolstov --- util/xpool/pool.go | 25 +++++++++++++++++++++++++ util/xpool/pool_test.go | 27 +++++++++++++++++++++++++++ 2 files changed, 52 insertions(+) create mode 100644 util/xpool/pool.go create mode 100644 util/xpool/pool_test.go diff --git a/util/xpool/pool.go b/util/xpool/pool.go new file mode 100644 index 00000000..1ffe4293 --- /dev/null +++ b/util/xpool/pool.go @@ -0,0 +1,25 @@ +package pool + +import "sync" + +type Pool[T any] struct { + p *sync.Pool +} + +func NewPool[T any](fn func() T) Pool[T] { + return Pool[T]{ + p: &sync.Pool{ + New: func() interface{} { + return fn() + }, + }, + } +} + +func (p Pool[T]) Get() T { + return p.p.Get().(T) +} + +func (p Pool[T]) Put(t T) { + p.p.Put(t) +} diff --git a/util/xpool/pool_test.go b/util/xpool/pool_test.go new file mode 100644 index 00000000..8e7a9b81 --- /dev/null +++ b/util/xpool/pool_test.go @@ -0,0 +1,27 @@ +package pool + +import ( + "bytes" + "strings" + "testing" +) + +func TestBytes(t *testing.T) { + p := NewPool(func() *bytes.Buffer { return bytes.NewBuffer(nil) }) + b := p.Get() + b.Write([]byte(`test`)) + if b.String() != "test" { + t.Fatal("pool not works") + } + p.Put(b) +} + +func TestStrings(t *testing.T) { + p := NewPool(func() *strings.Builder { return &strings.Builder{} }) + b := p.Get() + b.Write([]byte(`test`)) + if b.String() != "test" { + t.Fatal("pool not works") + } + p.Put(b) +} From 7b7cf18a6582c37933c7f73733ec47fa850bb3c5 Mon Sep 17 00:00:00 2001 From: Vasiliy Tolstov Date: Sun, 14 Apr 2024 16:42:36 +0300 Subject: [PATCH 09/59] semconv: add cache metric names Signed-off-by: Vasiliy Tolstov --- semconv/cache.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 semconv/cache.go diff --git a/semconv/cache.go b/semconv/cache.go new file mode 100644 index 00000000..f12739c1 --- /dev/null +++ b/semconv/cache.go @@ -0,0 +1,12 @@ +package semconv + +var ( + // CacheRequestDurationSeconds specifies meter metric name + CacheRequestDurationSeconds = "cache_request_duration_seconds" + // ClientRequestLatencyMicroseconds specifies meter metric name + CacheRequestLatencyMicroseconds = "cache_request_latency_microseconds" + // CacheRequestTotal specifies meter metric name + CacheRequestTotal = "cache_request_total" + // CacheRequestInflight specifies meter metric name + CacheRequestInflight = "cache_request_inflight" +) From c3de003e4a74769c454c3c6662c2f03c92c40aa8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=9A=D0=B8=D1=80=D0=B8=D0=BB=D0=BB=20=D0=93=D0=BE=D1=80?= =?UTF-8?q?=D0=B1=D1=83=D0=BD=D0=BE=D0=B2?= Date: Mon, 15 Apr 2024 13:30:48 +0300 Subject: [PATCH 10/59] #335 caller skip count. (#337) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Gorbunov Kirill Andreevich Reviewed-on: https://git.unistack.org/unistack-org/micro/pulls/337 Co-authored-by: Кирилл Горбунов Co-committed-by: Кирилл Горбунов --- logger/options.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/logger/options.go b/logger/options.go index 4d083d81..3138892d 100644 --- a/logger/options.go +++ b/logger/options.go @@ -182,3 +182,10 @@ func WithMicroKeys() Option { o.ErrorKey = "error" } } + +// WithAddCallerSkipCount add skip count for copy logger +func WithAddCallerSkipCount(n int) Option { + return func(o *Options) { + o.CallerSkipCount += n + } +} From add3ce478ca823c35f012171504d5d4f8c85a128 Mon Sep 17 00:00:00 2001 From: Vasiliy Tolstov Date: Mon, 22 Apr 2024 08:47:50 +0300 Subject: [PATCH 11/59] replace wrappers with hooks Signed-off-by: Vasiliy Tolstov --- broker/broker.go | 11 + broker/memory/memory.go | 46 +++- broker/memory/memory_test.go | 8 + broker/noop.go | 58 +++- broker/noop_test.go | 35 +++ broker/options.go | 15 +- client/client.go | 11 + client/client_call_options_test.go | 26 -- client/noop.go | 74 +++-- client/noop_test.go | 35 +++ client/options.go | 36 +-- config/config.go | 7 + config/default.go | 40 ++- config/default_test.go | 29 ++ config/options.go | 10 + options/hooks.go | 2 +- server/handler.go | 59 ---- server/noop.go | 425 +++++++++++++++++++++++++++++ server/options.go | 37 +-- server/request.go | 35 --- server/server.go | 16 +- server/subscriber.go | 345 +---------------------- store/{ => memory}/memory.go | 91 ++++-- store/{ => memory}/memory_test.go | 45 ++- store/noop.go | 147 ++++++++++ store/noop_test.go | 35 +++ store/options.go | 11 +- store/store.go | 14 + 28 files changed, 1116 insertions(+), 587 deletions(-) create mode 100644 broker/noop_test.go delete mode 100644 client/client_call_options_test.go create mode 100644 client/noop_test.go delete mode 100644 server/handler.go delete mode 100644 server/request.go rename store/{ => memory}/memory.go (59%) rename store/{ => memory}/memory_test.go (66%) create mode 100644 store/noop.go create mode 100644 store/noop_test.go diff --git a/broker/broker.go b/broker/broker.go index 80585c4c..f716c947 100644 --- a/broker/broker.go +++ b/broker/broker.go @@ -48,6 +48,17 @@ type Broker interface { String() string } +type ( + FuncPublish func(ctx context.Context, topic string, msg *Message, opts ...PublishOption) error + HookPublish func(next FuncPublish) FuncPublish + FuncBatchPublish func(ctx context.Context, msgs []*Message, opts ...PublishOption) error + HookBatchPublish func(next FuncBatchPublish) FuncBatchPublish + FuncSubscribe func(ctx context.Context, topic string, h Handler, opts ...SubscribeOption) (Subscriber, error) + HookSubscribe func(next FuncSubscribe) FuncSubscribe + FuncBatchSubscribe func(ctx context.Context, topic string, h BatchHandler, opts ...SubscribeOption) (Subscriber, error) + HookBatchSubscribe func(next FuncBatchSubscribe) FuncBatchSubscribe +) + // Handler is used to process messages via a subscription of a topic. type Handler func(Event) error diff --git a/broker/memory/memory.go b/broker/memory/memory.go index d5609e1e..1b301baf 100644 --- a/broker/memory/memory.go +++ b/broker/memory/memory.go @@ -7,6 +7,7 @@ import ( "go.unistack.org/micro/v3/broker" "go.unistack.org/micro/v3/logger" "go.unistack.org/micro/v3/metadata" + "go.unistack.org/micro/v3/options" maddr "go.unistack.org/micro/v3/util/addr" "go.unistack.org/micro/v3/util/id" mnet "go.unistack.org/micro/v3/util/net" @@ -14,9 +15,13 @@ import ( ) type memoryBroker struct { - subscribers map[string][]*memorySubscriber - addr string - opts broker.Options + funcPublish broker.FuncPublish + funcBatchPublish broker.FuncBatchPublish + funcSubscribe broker.FuncSubscribe + funcBatchSubscribe broker.FuncBatchSubscribe + subscribers map[string][]*memorySubscriber + addr string + opts broker.Options sync.RWMutex connected bool } @@ -98,15 +103,42 @@ func (m *memoryBroker) Init(opts ...broker.Option) error { for _, o := range opts { o(&m.opts) } + + m.funcPublish = m.fnPublish + m.funcBatchPublish = m.fnBatchPublish + m.funcSubscribe = m.fnSubscribe + m.funcBatchSubscribe = m.fnBatchSubscribe + + m.opts.Hooks.EachNext(func(hook options.Hook) { + switch h := hook.(type) { + case broker.HookPublish: + m.funcPublish = h(m.funcPublish) + case broker.HookBatchPublish: + m.funcBatchPublish = h(m.funcBatchPublish) + case broker.HookSubscribe: + m.funcSubscribe = h(m.funcSubscribe) + case broker.HookBatchSubscribe: + m.funcBatchSubscribe = h(m.funcBatchSubscribe) + } + }) + return nil } func (m *memoryBroker) Publish(ctx context.Context, topic string, msg *broker.Message, opts ...broker.PublishOption) error { + return m.funcPublish(ctx, topic, msg, opts...) +} + +func (m *memoryBroker) fnPublish(ctx context.Context, topic string, msg *broker.Message, opts ...broker.PublishOption) error { msg.Header.Set(metadata.HeaderTopic, topic) return m.publish(ctx, []*broker.Message{msg}, opts...) } func (m *memoryBroker) BatchPublish(ctx context.Context, msgs []*broker.Message, opts ...broker.PublishOption) error { + return m.funcBatchPublish(ctx, msgs, opts...) +} + +func (m *memoryBroker) fnBatchPublish(ctx context.Context, msgs []*broker.Message, opts ...broker.PublishOption) error { return m.publish(ctx, msgs, opts...) } @@ -202,6 +234,10 @@ func (m *memoryBroker) publish(ctx context.Context, msgs []*broker.Message, opts } func (m *memoryBroker) BatchSubscribe(ctx context.Context, topic string, handler broker.BatchHandler, opts ...broker.SubscribeOption) (broker.Subscriber, error) { + return m.funcBatchSubscribe(ctx, topic, handler, opts...) +} + +func (m *memoryBroker) fnBatchSubscribe(ctx context.Context, topic string, handler broker.BatchHandler, opts ...broker.SubscribeOption) (broker.Subscriber, error) { m.RLock() if !m.connected { m.RUnlock() @@ -247,6 +283,10 @@ func (m *memoryBroker) BatchSubscribe(ctx context.Context, topic string, handler } func (m *memoryBroker) Subscribe(ctx context.Context, topic string, handler broker.Handler, opts ...broker.SubscribeOption) (broker.Subscriber, error) { + return m.funcSubscribe(ctx, topic, handler, opts...) +} + +func (m *memoryBroker) fnSubscribe(ctx context.Context, topic string, handler broker.Handler, opts ...broker.SubscribeOption) (broker.Subscriber, error) { m.RLock() if !m.connected { m.RUnlock() diff --git a/broker/memory/memory_test.go b/broker/memory/memory_test.go index e558ef10..7557e6b0 100644 --- a/broker/memory/memory_test.go +++ b/broker/memory/memory_test.go @@ -13,6 +13,10 @@ func TestMemoryBatchBroker(t *testing.T) { b := NewBroker() ctx := context.Background() + if err := b.Init(); err != nil { + t.Fatalf("Unexpected init error %v", err) + } + if err := b.Connect(ctx); err != nil { t.Fatalf("Unexpected connect error %v", err) } @@ -59,6 +63,10 @@ func TestMemoryBroker(t *testing.T) { b := NewBroker() ctx := context.Background() + if err := b.Init(); err != nil { + t.Fatalf("Unexpected init error %v", err) + } + if err := b.Connect(ctx); err != nil { t.Fatalf("Unexpected connect error %v", err) } diff --git a/broker/noop.go b/broker/noop.go index 29142673..43f6e7dc 100644 --- a/broker/noop.go +++ b/broker/noop.go @@ -3,14 +3,25 @@ package broker import ( "context" "strings" + + "go.unistack.org/micro/v3/options" ) type NoopBroker struct { - opts Options + funcPublish FuncPublish + funcBatchPublish FuncBatchPublish + funcSubscribe FuncSubscribe + funcBatchSubscribe FuncBatchSubscribe + opts Options } func NewBroker(opts ...Option) *NoopBroker { b := &NoopBroker{opts: NewOptions(opts...)} + b.funcPublish = b.fnPublish + b.funcBatchPublish = b.fnBatchPublish + b.funcSubscribe = b.fnSubscribe + b.funcBatchSubscribe = b.fnBatchSubscribe + return b } @@ -30,6 +41,25 @@ func (b *NoopBroker) Init(opts ...Option) error { for _, opt := range opts { opt(&b.opts) } + + b.funcPublish = b.fnPublish + b.funcBatchPublish = b.fnBatchPublish + b.funcSubscribe = b.fnSubscribe + b.funcBatchSubscribe = b.fnBatchSubscribe + + b.opts.Hooks.EachNext(func(hook options.Hook) { + switch h := hook.(type) { + case HookPublish: + b.funcPublish = h(b.funcPublish) + case HookBatchPublish: + b.funcBatchPublish = h(b.funcBatchPublish) + case HookSubscribe: + b.funcSubscribe = h(b.funcSubscribe) + case HookBatchSubscribe: + b.funcBatchSubscribe = h(b.funcBatchSubscribe) + } + }) + return nil } @@ -45,14 +75,22 @@ func (b *NoopBroker) Address() string { return strings.Join(b.opts.Addrs, ",") } -func (b *NoopBroker) BatchPublish(_ context.Context, _ []*Message, _ ...PublishOption) error { +func (b *NoopBroker) fnBatchPublish(_ context.Context, _ []*Message, _ ...PublishOption) error { return nil } -func (b *NoopBroker) Publish(_ context.Context, _ string, _ *Message, _ ...PublishOption) error { +func (b *NoopBroker) BatchPublish(ctx context.Context, msgs []*Message, opts ...PublishOption) error { + return b.funcBatchPublish(ctx, msgs, opts...) +} + +func (b *NoopBroker) fnPublish(_ context.Context, _ string, _ *Message, _ ...PublishOption) error { return nil } +func (b *NoopBroker) Publish(ctx context.Context, topic string, msg *Message, opts ...PublishOption) error { + return b.funcPublish(ctx, topic, msg, opts...) +} + type NoopSubscriber struct { ctx context.Context topic string @@ -61,14 +99,22 @@ type NoopSubscriber struct { opts SubscribeOptions } -func (b *NoopBroker) BatchSubscribe(ctx context.Context, topic string, handler BatchHandler, opts ...SubscribeOption) (Subscriber, error) { +func (b *NoopBroker) fnBatchSubscribe(ctx context.Context, topic string, handler BatchHandler, opts ...SubscribeOption) (Subscriber, error) { return &NoopSubscriber{ctx: ctx, topic: topic, opts: NewSubscribeOptions(opts...), batchHandler: handler}, nil } -func (b *NoopBroker) Subscribe(ctx context.Context, topic string, handler Handler, opts ...SubscribeOption) (Subscriber, error) { +func (b *NoopBroker) BatchSubscribe(ctx context.Context, topic string, handler BatchHandler, opts ...SubscribeOption) (Subscriber, error) { + return b.funcBatchSubscribe(ctx, topic, handler, opts...) +} + +func (b *NoopBroker) fnSubscribe(ctx context.Context, topic string, handler Handler, opts ...SubscribeOption) (Subscriber, error) { return &NoopSubscriber{ctx: ctx, topic: topic, opts: NewSubscribeOptions(opts...), handler: handler}, nil } +func (b *NoopBroker) Subscribe(ctx context.Context, topic string, handler Handler, opts ...SubscribeOption) (Subscriber, error) { + return b.funcSubscribe(ctx, topic, handler, opts...) +} + func (s *NoopSubscriber) Options() SubscribeOptions { return s.opts } @@ -77,6 +123,6 @@ func (s *NoopSubscriber) Topic() string { return s.topic } -func (s *NoopSubscriber) Unsubscribe(ctx context.Context) error { +func (s *NoopSubscriber) Unsubscribe(_ context.Context) error { return nil } diff --git a/broker/noop_test.go b/broker/noop_test.go new file mode 100644 index 00000000..824b4b00 --- /dev/null +++ b/broker/noop_test.go @@ -0,0 +1,35 @@ +package broker + +import ( + "context" + "testing" +) + +type testHook struct { + f bool +} + +func (t *testHook) Publish1(fn FuncPublish) FuncPublish { + return func(ctx context.Context, topic string, msg *Message, opts ...PublishOption) error { + t.f = true + return fn(ctx, topic, msg, opts...) + } +} + +func TestNoopHook(t *testing.T) { + h := &testHook{} + + b := NewBroker(Hooks(HookPublish(h.Publish1))) + + if err := b.Init(); err != nil { + t.Fatal(err) + } + + if err := b.Publish(context.TODO(), "", nil); err != nil { + t.Fatal(err) + } + + if !h.f { + t.Fatal("hook not works") + } +} diff --git a/broker/options.go b/broker/options.go index c5e91263..7e26adbc 100644 --- a/broker/options.go +++ b/broker/options.go @@ -8,6 +8,7 @@ import ( "go.unistack.org/micro/v3/codec" "go.unistack.org/micro/v3/logger" "go.unistack.org/micro/v3/meter" + "go.unistack.org/micro/v3/options" "go.unistack.org/micro/v3/register" "go.unistack.org/micro/v3/sync" "go.unistack.org/micro/v3/tracer" @@ -37,10 +38,13 @@ type Options struct { Name string // Addrs holds the broker address Addrs []string - + // Wait waits for a collection of goroutines to finish Wait *sync.WaitGroup - + // GracefulTimeout contains time to wait to finish in flight requests GracefulTimeout time.Duration + // Hooks can be run before broker Publish/BatchPublish and + // Subscribe/BatchSubscribe methods + Hooks options.Hooks } // NewOptions create new Options @@ -230,6 +234,13 @@ func Name(n string) Option { } } +// Hooks sets hook runs before action +func Hooks(h ...options.Hook) Option { + return func(o *Options) { + o.Hooks = append(o.Hooks, h...) + } +} + // SubscribeContext set context func SubscribeContext(ctx context.Context) SubscribeOption { return func(o *SubscribeOptions) { diff --git a/client/client.go b/client/client.go index 6d60aef0..e7c42692 100644 --- a/client/client.go +++ b/client/client.go @@ -44,6 +44,17 @@ type Client interface { String() string } +type ( + FuncCall func(ctx context.Context, req Request, rsp interface{}, opts ...CallOption) error + HookCall func(next FuncCall) FuncCall + FuncStream func(ctx context.Context, req Request, opts ...CallOption) (Stream, error) + HookStream func(next FuncStream) FuncStream + FuncPublish func(ctx context.Context, msg Message, opts ...PublishOption) error + HookPublish func(next FuncPublish) FuncPublish + FuncBatchPublish func(ctx context.Context, msg []Message, opts ...PublishOption) error + HookBatchPublish func(next FuncBatchPublish) FuncBatchPublish +) + // Message is the interface for publishing asynchronously type Message interface { Topic() string diff --git a/client/client_call_options_test.go b/client/client_call_options_test.go deleted file mode 100644 index 2cffdf36..00000000 --- a/client/client_call_options_test.go +++ /dev/null @@ -1,26 +0,0 @@ -package client - -import ( - "context" - "testing" - "time" -) - -func TestNewClientCallOptions(t *testing.T) { - var flag bool - w := func(fn CallFunc) CallFunc { - flag = true - return fn - } - c := NewClientCallOptions(NewClient(), - WithAddress("127.0.0.1"), - WithCallWrapper(w), - WithRequestTimeout(1*time.Millisecond), - WithRetries(0), - WithBackoff(BackoffInterval(10*time.Millisecond, 100*time.Millisecond)), - ) - _ = c.Call(context.TODO(), c.NewRequest("service", "endpoint", nil), nil) - if !flag { - t.Fatalf("NewClientCallOptions not works") - } -} diff --git a/client/noop.go b/client/noop.go index 1c92ed71..b5ae7791 100644 --- a/client/noop.go +++ b/client/noop.go @@ -10,6 +10,7 @@ import ( "go.unistack.org/micro/v3/codec" "go.unistack.org/micro/v3/errors" "go.unistack.org/micro/v3/metadata" + "go.unistack.org/micro/v3/options" "go.unistack.org/micro/v3/selector" ) @@ -19,7 +20,11 @@ var DefaultCodecs = map[string]codec.Codec{ } type noopClient struct { - opts Options + funcPublish FuncPublish + funcBatchPublish FuncBatchPublish + funcCall FuncCall + funcStream FuncStream + opts Options } type noopMessage struct { @@ -40,16 +45,14 @@ type noopRequest struct { // NewClient returns new noop client func NewClient(opts ...Option) Client { - nc := &noopClient{opts: NewOptions(opts...)} - // wrap in reverse + n := &noopClient{opts: NewOptions(opts...)} - c := Client(nc) + n.funcCall = n.fnCall + n.funcStream = n.fnStream + n.funcPublish = n.fnPublish + n.funcBatchPublish = n.fnBatchPublish - for i := len(nc.opts.Wrappers); i > 0; i-- { - c = nc.opts.Wrappers[i-1](c) - } - - return c + return n } func (n *noopClient) Name() string { @@ -173,6 +176,25 @@ func (n *noopClient) Init(opts ...Option) error { for _, o := range opts { o(&n.opts) } + + n.funcCall = n.fnCall + n.funcStream = n.fnStream + n.funcPublish = n.fnPublish + n.funcBatchPublish = n.fnBatchPublish + + n.opts.Hooks.EachNext(func(hook options.Hook) { + switch h := hook.(type) { + case HookCall: + n.funcCall = h(n.funcCall) + case HookStream: + n.funcStream = h(n.funcStream) + case HookPublish: + n.funcPublish = h(n.funcPublish) + case HookBatchPublish: + n.funcBatchPublish = h(n.funcBatchPublish) + } + }) + return nil } @@ -185,6 +207,10 @@ func (n *noopClient) String() string { } func (n *noopClient) Call(ctx context.Context, req Request, rsp interface{}, opts ...CallOption) error { + return n.funcCall(ctx, req, rsp, opts...) +} + +func (n *noopClient) fnCall(ctx context.Context, req Request, rsp interface{}, opts ...CallOption) error { // make a copy of call opts callOpts := n.opts.CallOptions for _, opt := range opts { @@ -213,11 +239,8 @@ func (n *noopClient) Call(ctx context.Context, req Request, rsp interface{}, opt } // make copy of call method - hcall := n.call - - // wrap the call in reverse - for i := len(callOpts.CallWrappers); i > 0; i-- { - hcall = callOpts.CallWrappers[i-1](hcall) + hcall := func(ctx context.Context, addr string, req Request, rsp interface{}, opts CallOptions) error { + return nil } // use the router passed as a call option, or fallback to the rpc clients router @@ -316,10 +339,6 @@ func (n *noopClient) Call(ctx context.Context, req Request, rsp interface{}, opt return gerr } -func (n *noopClient) call(ctx context.Context, addr string, req Request, rsp interface{}, opts CallOptions) error { - return nil -} - func (n *noopClient) NewRequest(service, endpoint string, req interface{}, opts ...RequestOption) Request { return &noopRequest{service: service, endpoint: endpoint} } @@ -330,6 +349,10 @@ func (n *noopClient) NewMessage(topic string, msg interface{}, opts ...MessageOp } func (n *noopClient) Stream(ctx context.Context, req Request, opts ...CallOption) (Stream, error) { + return n.funcStream(ctx, req, opts...) +} + +func (n *noopClient) fnStream(ctx context.Context, req Request, opts ...CallOption) (Stream, error) { var err error // make a copy of call opts @@ -474,10 +497,18 @@ func (n *noopClient) stream(ctx context.Context, addr string, req Request, opts } func (n *noopClient) BatchPublish(ctx context.Context, ps []Message, opts ...PublishOption) error { + return n.funcBatchPublish(ctx, ps, opts...) +} + +func (n *noopClient) fnBatchPublish(ctx context.Context, ps []Message, opts ...PublishOption) error { return n.publish(ctx, ps, opts...) } func (n *noopClient) Publish(ctx context.Context, p Message, opts ...PublishOption) error { + return n.funcPublish(ctx, p, opts...) +} + +func (n *noopClient) fnPublish(ctx context.Context, p Message, opts ...PublishOption) error { return n.publish(ctx, []Message{p}, opts...) } @@ -538,6 +569,13 @@ func (n *noopClient) publish(ctx context.Context, ps []Message, opts ...PublishO msgs = append(msgs, &broker.Message{Header: md, Body: body}) } + if len(msgs) == 1 { + return n.opts.Broker.Publish(ctx, msgs[0].Header[metadata.HeaderTopic], msgs[0], + broker.PublishContext(options.Context), + broker.PublishBodyOnly(options.BodyOnly), + ) + } + return n.opts.Broker.BatchPublish(ctx, msgs, broker.PublishContext(options.Context), broker.PublishBodyOnly(options.BodyOnly), diff --git a/client/noop_test.go b/client/noop_test.go new file mode 100644 index 00000000..cc05204c --- /dev/null +++ b/client/noop_test.go @@ -0,0 +1,35 @@ +package client + +import ( + "context" + "testing" +) + +type testHook struct { + f bool +} + +func (t *testHook) Publish(fn FuncPublish) FuncPublish { + return func(ctx context.Context, msg Message, opts ...PublishOption) error { + t.f = true + return fn(ctx, msg, opts...) + } +} + +func TestNoopHook(t *testing.T) { + h := &testHook{} + + c := NewClient(Hooks(HookPublish(h.Publish))) + + if err := c.Init(); err != nil { + t.Fatal(err) + } + + if err := c.Publish(context.TODO(), c.NewMessage("", nil, MessageContentType("application/octet-stream"))); err != nil { + t.Fatal(err) + } + + if !h.f { + t.Fatal("hook not works") + } +} diff --git a/client/options.go b/client/options.go index d4f976d4..6c8d0637 100644 --- a/client/options.go +++ b/client/options.go @@ -12,6 +12,7 @@ import ( "go.unistack.org/micro/v3/metadata" "go.unistack.org/micro/v3/meter" "go.unistack.org/micro/v3/network/transport" + "go.unistack.org/micro/v3/options" "go.unistack.org/micro/v3/register" "go.unistack.org/micro/v3/router" "go.unistack.org/micro/v3/selector" @@ -59,6 +60,9 @@ type Options struct { PoolTTL time.Duration // ContextDialer used to connect ContextDialer func(context.Context, string) (net.Conn, error) + // Hooks can be run before broker Publish/BatchPublish and + // Subscribe/BatchSubscribe methods + Hooks options.Hooks } // NewCallOptions creates new call options struct @@ -92,8 +96,6 @@ type CallOptions struct { Address []string // SelectOptions selector options SelectOptions []selector.SelectOption - // CallWrappers call wrappers - CallWrappers []CallWrapper // StreamTimeout stream timeout StreamTimeout time.Duration // RequestTimeout request timeout @@ -185,7 +187,7 @@ func NewOptions(opts ...Option) Options { options := Options{ Context: context.Background(), ContentType: DefaultContentType, - Codecs: make(map[string]codec.Codec), + Codecs: DefaultCodecs, CallOptions: CallOptions{ Context: context.Background(), Backoff: DefaultBackoff, @@ -306,20 +308,6 @@ func Selector(s selector.Selector) Option { } } -// Wrap adds a wrapper to the list of options passed into the client -func Wrap(w Wrapper) Option { - return func(o *Options) { - o.Wrappers = append(o.Wrappers, w) - } -} - -// WrapCall adds a wrapper to the list of CallFunc wrappers -func WrapCall(cw ...CallWrapper) Option { - return func(o *Options) { - o.CallOptions.CallWrappers = append(o.CallOptions.CallWrappers, cw...) - } -} - // Backoff is used to set the backoff function used when retrying Calls func Backoff(fn BackoffFunc) Option { return func(o *Options) { @@ -450,13 +438,6 @@ func WithAddress(a ...string) CallOption { } } -// WithCallWrapper is a CallOption which adds to the existing CallFunc wrappers -func WithCallWrapper(cw ...CallWrapper) CallOption { - return func(o *CallOptions) { - o.CallWrappers = append(o.CallWrappers, cw...) - } -} - // WithBackoff is a CallOption which overrides that which // set in Options.CallOptions func WithBackoff(fn BackoffFunc) CallOption { @@ -591,3 +572,10 @@ func RequestContentType(ct string) RequestOption { o.ContentType = ct } } + +// Hooks sets hook runs before action +func Hooks(h ...options.Hook) Option { + return func(o *Options) { + o.Hooks = append(o.Hooks, h...) + } +} diff --git a/config/config.go b/config/config.go index 44c18f76..955f2957 100644 --- a/config/config.go +++ b/config/config.go @@ -50,6 +50,13 @@ type Config interface { String() string } +type ( + FuncLoad func(ctx context.Context, opts ...LoadOption) error + HookLoad func(next FuncLoad) FuncLoad + FuncSave func(ctx context.Context, opts ...SaveOption) error + HookSave func(next FuncSave) FuncSave +) + // Watcher is the config watcher type Watcher interface { // Next blocks until update happens or error returned diff --git a/config/default.go b/config/default.go index c03fb812..62cef02c 100644 --- a/config/default.go +++ b/config/default.go @@ -9,13 +9,16 @@ import ( "dario.cat/mergo" "github.com/google/uuid" + "go.unistack.org/micro/v3/options" mid "go.unistack.org/micro/v3/util/id" rutil "go.unistack.org/micro/v3/util/reflect" mtime "go.unistack.org/micro/v3/util/time" ) type defaultConfig struct { - opts Options + funcLoad FuncLoad + funcSave FuncSave + opts Options } func (c *defaultConfig) Options() Options { @@ -31,6 +34,18 @@ func (c *defaultConfig) Init(opts ...Option) error { return err } + c.funcLoad = c.fnLoad + c.funcSave = c.fnSave + + c.opts.Hooks.EachNext(func(hook options.Hook) { + switch h := hook.(type) { + case HookLoad: + c.funcLoad = h(c.funcLoad) + case HookSave: + c.funcSave = h(c.funcSave) + } + }) + if err := DefaultAfterInit(c.opts.Context, c); err != nil && !c.opts.AllowFail { return err } @@ -39,11 +54,17 @@ func (c *defaultConfig) Init(opts ...Option) error { } func (c *defaultConfig) Load(ctx context.Context, opts ...LoadOption) error { + return c.funcLoad(ctx, opts...) +} + +func (c *defaultConfig) fnLoad(ctx context.Context, opts ...LoadOption) error { + var err error + if c.opts.SkipLoad != nil && c.opts.SkipLoad(ctx, c) { return nil } - if err := DefaultBeforeLoad(ctx, c); err != nil && !c.opts.AllowFail { + if err = DefaultBeforeLoad(ctx, c); err != nil && !c.opts.AllowFail { return err } @@ -233,6 +254,7 @@ func fillValue(value reflect.Value, val string) error { } value.Set(reflect.ValueOf(v)) } + return nil } @@ -295,7 +317,11 @@ func fillValues(valueOf reflect.Value, tname string) error { return nil } -func (c *defaultConfig) Save(ctx context.Context, _ ...SaveOption) error { +func (c *defaultConfig) Save(ctx context.Context, opts ...SaveOption) error { + return c.funcSave(ctx, opts...) +} + +func (c *defaultConfig) fnSave(ctx context.Context, opts ...SaveOption) error { if c.opts.SkipSave != nil && c.opts.SkipSave(ctx, c) { return nil } @@ -319,7 +345,7 @@ func (c *defaultConfig) Name() string { return c.opts.Name } -func (c *defaultConfig) Watch(ctx context.Context, opts ...WatchOption) (Watcher, error) { +func (c *defaultConfig) Watch(_ context.Context, _ ...WatchOption) (Watcher, error) { return nil, ErrWatcherNotImplemented } @@ -329,5 +355,9 @@ func NewConfig(opts ...Option) Config { if len(options.StructTag) == 0 { options.StructTag = "default" } - return &defaultConfig{opts: options} + c := &defaultConfig{opts: options} + c.funcLoad = c.fnLoad + c.funcSave = c.fnSave + + return c } diff --git a/config/default_test.go b/config/default_test.go index 40ee6924..a4d90650 100644 --- a/config/default_test.go +++ b/config/default_test.go @@ -41,6 +41,35 @@ func (c *cfgStructValue) Validate() error { return nil } +type testHook struct { + f bool +} + +func (t *testHook) Load(fn config.FuncLoad) config.FuncLoad { + return func(ctx context.Context, opts ...config.LoadOption) error { + t.f = true + return fn(ctx, opts...) + } +} + +func TestHook(t *testing.T) { + h := &testHook{} + + c := config.NewConfig(config.Struct(h), config.Hooks(config.HookLoad(h.Load))) + + if err := c.Init(); err != nil { + t.Fatal(err) + } + + if err := c.Load(context.TODO()); err != nil { + t.Fatal(err) + } + + if !h.f { + t.Fatal("hook not works") + } +} + func TestDefault(t *testing.T) { ctx := context.Background() conf := &cfg{IntValue: 10} diff --git a/config/options.go b/config/options.go index 38a3fbec..62e57659 100644 --- a/config/options.go +++ b/config/options.go @@ -7,6 +7,7 @@ import ( "go.unistack.org/micro/v3/codec" "go.unistack.org/micro/v3/logger" "go.unistack.org/micro/v3/meter" + "go.unistack.org/micro/v3/options" "go.unistack.org/micro/v3/tracer" ) @@ -46,6 +47,8 @@ type Options struct { SkipLoad func(context.Context, Config) bool // SkipSave runs only if condition returns true SkipSave func(context.Context, Config) bool + // Hooks can be run before/after config Save/Load + Hooks options.Hooks } // Option function signature @@ -288,3 +291,10 @@ func WatchStruct(src interface{}) WatchOption { o.Struct = src } } + +// Hooks sets hook runs before action +func Hooks(h ...options.Hook) Option { + return func(o *Options) { + o.Hooks = append(o.Hooks, h...) + } +} diff --git a/options/hooks.go b/options/hooks.go index 48a45ad2..99e66f40 100644 --- a/options/hooks.go +++ b/options/hooks.go @@ -1,4 +1,4 @@ -package options // import "go.unistack.org/micro/v3/options" +package options // Hook func interface type Hook interface{} diff --git a/server/handler.go b/server/handler.go deleted file mode 100644 index 8b99a0ff..00000000 --- a/server/handler.go +++ /dev/null @@ -1,59 +0,0 @@ -package server - -import ( - "reflect" - - "go.unistack.org/micro/v3/register" -) - -type rpcHandler struct { - opts HandlerOptions - handler interface{} - name string - endpoints []*register.Endpoint -} - -func newRPCHandler(handler interface{}, opts ...HandlerOption) Handler { - options := NewHandlerOptions(opts...) - - typ := reflect.TypeOf(handler) - hdlr := reflect.ValueOf(handler) - name := reflect.Indirect(hdlr).Type().Name() - - var endpoints []*register.Endpoint - - for m := 0; m < typ.NumMethod(); m++ { - if e := register.ExtractEndpoint(typ.Method(m)); e != nil { - e.Name = name + "." + e.Name - - for k, v := range options.Metadata[e.Name] { - e.Metadata[k] = v - } - - endpoints = append(endpoints, e) - } - } - - return &rpcHandler{ - name: name, - handler: handler, - endpoints: endpoints, - opts: options, - } -} - -func (r *rpcHandler) Name() string { - return r.name -} - -func (r *rpcHandler) Handler() interface{} { - return r.handler -} - -func (r *rpcHandler) Endpoints() []*register.Endpoint { - return r.endpoints -} - -func (r *rpcHandler) Options() HandlerOptions { - return r.opts -} diff --git a/server/noop.go b/server/noop.go index c397eee8..f2bc031f 100644 --- a/server/noop.go +++ b/server/noop.go @@ -1,14 +1,22 @@ package server import ( + "bytes" + "context" "fmt" + "reflect" + "runtime/debug" "sort" + "strings" "sync" "time" "go.unistack.org/micro/v3/broker" "go.unistack.org/micro/v3/codec" + "go.unistack.org/micro/v3/errors" "go.unistack.org/micro/v3/logger" + "go.unistack.org/micro/v3/metadata" + "go.unistack.org/micro/v3/options" "go.unistack.org/micro/v3/register" maddr "go.unistack.org/micro/v3/util/addr" mnet "go.unistack.org/micro/v3/util/net" @@ -24,6 +32,58 @@ const ( defaultContentType = "application/json" ) +type rpcHandler struct { + opts HandlerOptions + handler interface{} + name string + endpoints []*register.Endpoint +} + +func newRPCHandler(handler interface{}, opts ...HandlerOption) Handler { + options := NewHandlerOptions(opts...) + + typ := reflect.TypeOf(handler) + hdlr := reflect.ValueOf(handler) + name := reflect.Indirect(hdlr).Type().Name() + + var endpoints []*register.Endpoint + + for m := 0; m < typ.NumMethod(); m++ { + if e := register.ExtractEndpoint(typ.Method(m)); e != nil { + e.Name = name + "." + e.Name + + for k, v := range options.Metadata[e.Name] { + e.Metadata[k] = v + } + + endpoints = append(endpoints, e) + } + } + + return &rpcHandler{ + name: name, + handler: handler, + endpoints: endpoints, + opts: options, + } +} + +func (r *rpcHandler) Name() string { + return r.name +} + +func (r *rpcHandler) Handler() interface{} { + return r.handler +} + +func (r *rpcHandler) Endpoints() []*register.Endpoint { + return r.endpoints +} + +func (r *rpcHandler) Options() HandlerOptions { + return r.opts +} + type noopServer struct { h Handler wg *sync.WaitGroup @@ -94,6 +154,35 @@ func (n *noopServer) Subscribe(sb Subscriber) error { return nil } +type rpcMessage struct { + payload interface{} + codec codec.Codec + header metadata.Metadata + topic string + contentType string + body []byte +} + +func (r *rpcMessage) ContentType() string { + return r.contentType +} + +func (r *rpcMessage) Topic() string { + return r.topic +} + +func (r *rpcMessage) Body() interface{} { + return r.payload +} + +func (r *rpcMessage) Header() metadata.Metadata { + return r.header +} + +func (r *rpcMessage) Codec() codec.Codec { + return r.codec +} + func (n *noopServer) NewHandler(h interface{}, opts ...HandlerOption) Handler { return newRPCHandler(h, opts...) } @@ -478,3 +567,339 @@ func (n *noopServer) Stop() error { return err } + +func newSubscriber(topic string, sub interface{}, opts ...SubscriberOption) Subscriber { + var endpoints []*register.Endpoint + var handlers []*handler + + options := NewSubscriberOptions(opts...) + + if typ := reflect.TypeOf(sub); typ.Kind() == reflect.Func { + h := &handler{ + method: reflect.ValueOf(sub), + } + + switch typ.NumIn() { + case 1: + h.reqType = typ.In(0) + case 2: + h.ctxType = typ.In(0) + h.reqType = typ.In(1) + } + + handlers = append(handlers, h) + ep := ®ister.Endpoint{ + Name: "Func", + Request: register.ExtractSubValue(typ), + Metadata: metadata.New(2), + } + ep.Metadata.Set("topic", topic) + ep.Metadata.Set("subscriber", "true") + endpoints = append(endpoints, ep) + } else { + hdlr := reflect.ValueOf(sub) + name := reflect.Indirect(hdlr).Type().Name() + + for m := 0; m < typ.NumMethod(); m++ { + method := typ.Method(m) + h := &handler{ + method: method.Func, + } + + switch method.Type.NumIn() { + case 2: + h.reqType = method.Type.In(1) + case 3: + h.ctxType = method.Type.In(1) + h.reqType = method.Type.In(2) + } + + handlers = append(handlers, h) + ep := ®ister.Endpoint{ + Name: name + "." + method.Name, + Request: register.ExtractSubValue(method.Type), + Metadata: metadata.New(2), + } + ep.Metadata.Set("topic", topic) + ep.Metadata.Set("subscriber", "true") + endpoints = append(endpoints, ep) + } + } + + return &subscriber{ + rcvr: reflect.ValueOf(sub), + typ: reflect.TypeOf(sub), + topic: topic, + subscriber: sub, + handlers: handlers, + endpoints: endpoints, + opts: options, + } +} + +//nolint:gocyclo +func (n *noopServer) createBatchSubHandler(sb *subscriber, opts Options) broker.BatchHandler { + return func(ps broker.Events) (err error) { + defer func() { + if r := recover(); r != nil { + n.RLock() + config := n.opts + n.RUnlock() + if config.Logger.V(logger.ErrorLevel) { + config.Logger.Error(n.opts.Context, "panic recovered: ", r) + config.Logger.Error(n.opts.Context, string(debug.Stack())) + } + err = errors.InternalServerError(n.opts.Name+".subscriber", "panic recovered: %v", r) + } + }() + + msgs := make([]Message, 0, len(ps)) + ctxs := make([]context.Context, 0, len(ps)) + for _, p := range ps { + msg := p.Message() + // if we don't have headers, create empty map + if msg.Header == nil { + msg.Header = metadata.New(2) + } + + ct, _ := msg.Header.Get(metadata.HeaderContentType) + if len(ct) == 0 { + msg.Header.Set(metadata.HeaderContentType, defaultContentType) + ct = defaultContentType + } + hdr := metadata.Copy(msg.Header) + topic, _ := msg.Header.Get(metadata.HeaderTopic) + ctxs = append(ctxs, metadata.NewIncomingContext(sb.opts.Context, hdr)) + msgs = append(msgs, &rpcMessage{ + topic: topic, + contentType: ct, + header: msg.Header, + body: msg.Body, + }) + } + results := make(chan error, len(sb.handlers)) + + for i := 0; i < len(sb.handlers); i++ { + handler := sb.handlers[i] + + var req reflect.Value + + switch handler.reqType.Kind() { + case reflect.Ptr: + req = reflect.New(handler.reqType.Elem()) + default: + req = reflect.New(handler.reqType.Elem()).Elem() + } + + reqType := handler.reqType + var cf codec.Codec + for _, msg := range msgs { + cf, err = n.newCodec(msg.ContentType()) + if err != nil { + return err + } + rb := reflect.New(req.Type().Elem()) + if err = cf.ReadBody(bytes.NewReader(msg.(*rpcMessage).body), rb.Interface()); err != nil { + return err + } + msg.(*rpcMessage).codec = cf + msg.(*rpcMessage).payload = rb.Interface() + } + + fn := func(ctxs []context.Context, ms []Message) error { + var vals []reflect.Value + if sb.typ.Kind() != reflect.Func { + vals = append(vals, sb.rcvr) + } + if handler.ctxType != nil { + vals = append(vals, reflect.ValueOf(ctxs)) + } + payloads := reflect.MakeSlice(reqType, 0, len(ms)) + for _, m := range ms { + payloads = reflect.Append(payloads, reflect.ValueOf(m.Body())) + } + vals = append(vals, payloads) + + returnValues := handler.method.Call(vals) + if rerr := returnValues[0].Interface(); rerr != nil { + return rerr.(error) + } + return nil + } + + opts.Hooks.EachNext(func(hook options.Hook) { + if h, ok := hook.(HookBatchSubHandler); ok { + fn = h(fn) + } + }) + + if n.wg != nil { + n.wg.Add(1) + } + + go func() { + if n.wg != nil { + defer n.wg.Done() + } + results <- fn(ctxs, msgs) + }() + } + + var errors []string + for i := 0; i < len(sb.handlers); i++ { + if rerr := <-results; rerr != nil { + errors = append(errors, rerr.Error()) + } + } + if len(errors) > 0 { + err = fmt.Errorf("subscriber error: %s", strings.Join(errors, "\n")) + } + return err + } +} + +//nolint:gocyclo +func (n *noopServer) createSubHandler(sb *subscriber, opts Options) broker.Handler { + return func(p broker.Event) (err error) { + defer func() { + if r := recover(); r != nil { + n.RLock() + config := n.opts + n.RUnlock() + if config.Logger.V(logger.ErrorLevel) { + config.Logger.Error(n.opts.Context, "panic recovered: ", r) + config.Logger.Error(n.opts.Context, string(debug.Stack())) + } + err = errors.InternalServerError(n.opts.Name+".subscriber", "panic recovered: %v", r) + } + }() + + msg := p.Message() + // if we don't have headers, create empty map + if msg.Header == nil { + msg.Header = metadata.New(2) + } + + ct := msg.Header["Content-Type"] + if len(ct) == 0 { + msg.Header.Set(metadata.HeaderContentType, defaultContentType) + ct = defaultContentType + } + cf, err := n.newCodec(ct) + if err != nil { + return err + } + + hdr := metadata.New(len(msg.Header)) + for k, v := range msg.Header { + hdr.Set(k, v) + } + + ctx := metadata.NewIncomingContext(sb.opts.Context, hdr) + + results := make(chan error, len(sb.handlers)) + + for i := 0; i < len(sb.handlers); i++ { + handler := sb.handlers[i] + + var isVal bool + var req reflect.Value + + if handler.reqType.Kind() == reflect.Ptr { + req = reflect.New(handler.reqType.Elem()) + } else { + req = reflect.New(handler.reqType) + isVal = true + } + if isVal { + req = req.Elem() + } + + if err = cf.ReadBody(bytes.NewBuffer(msg.Body), req.Interface()); err != nil { + return err + } + + fn := func(ctx context.Context, msg Message) error { + var vals []reflect.Value + if sb.typ.Kind() != reflect.Func { + vals = append(vals, sb.rcvr) + } + if handler.ctxType != nil { + vals = append(vals, reflect.ValueOf(ctx)) + } + + vals = append(vals, reflect.ValueOf(msg.Body())) + + returnValues := handler.method.Call(vals) + if rerr := returnValues[0].Interface(); rerr != nil { + return rerr.(error) + } + return nil + } + + opts.Hooks.EachNext(func(hook options.Hook) { + if h, ok := hook.(HookSubHandler); ok { + fn = h(fn) + } + }) + + if n.wg != nil { + n.wg.Add(1) + } + go func() { + if n.wg != nil { + defer n.wg.Done() + } + cerr := fn(ctx, &rpcMessage{ + topic: sb.topic, + contentType: ct, + payload: req.Interface(), + header: msg.Header, + }) + results <- cerr + }() + } + var errors []string + for i := 0; i < len(sb.handlers); i++ { + if rerr := <-results; rerr != nil { + errors = append(errors, rerr.Error()) + } + } + if len(errors) > 0 { + err = fmt.Errorf("subscriber error: %s", strings.Join(errors, "\n")) + } + return err + } +} + +func (s *subscriber) Topic() string { + return s.topic +} + +func (s *subscriber) Subscriber() interface{} { + return s.subscriber +} + +func (s *subscriber) Endpoints() []*register.Endpoint { + return s.endpoints +} + +func (s *subscriber) Options() SubscriberOptions { + return s.opts +} + +type subscriber struct { + typ reflect.Type + subscriber interface{} + topic string + endpoints []*register.Endpoint + handlers []*handler + opts SubscriberOptions + rcvr reflect.Value +} + +type handler struct { + reqType reflect.Type + ctxType reflect.Type + method reflect.Value +} diff --git a/server/options.go b/server/options.go index e320ff86..093b5e0e 100644 --- a/server/options.go +++ b/server/options.go @@ -69,12 +69,6 @@ type Options struct { Advertise string // Version holds the server version Version string - // SubWrappers holds the server subscribe wrappers - SubWrappers []SubscriberWrapper - // BatchSubWrappers holds the server batch subscribe wrappers - BatchSubWrappers []BatchSubscriberWrapper - // HdlrWrappers holds the handler wrappers - HdlrWrappers []HandlerWrapper // RegisterAttempts holds the number of register attempts before error RegisterAttempts int // RegisterInterval holds he interval for re-register @@ -85,7 +79,8 @@ type Options struct { MaxConn int // DeregisterAttempts holds the number of deregister attempts before error DeregisterAttempts int - // Hooks may contains SubscriberWrapper, HandlerWrapper or Server func wrapper + // Hooks may contains hook actions that performs before/after server handler + // or server subscriber handler Hooks options.Hooks // GracefulTimeout timeout for graceful stop server GracefulTimeout time.Duration @@ -287,27 +282,6 @@ func Wait(wg *sync.WaitGroup) Option { } } -// WrapHandler adds a handler Wrapper to a list of options passed into the server -func WrapHandler(w HandlerWrapper) Option { - return func(o *Options) { - o.HdlrWrappers = append(o.HdlrWrappers, w) - } -} - -// WrapSubscriber adds a subscriber Wrapper to a list of options passed into the server -func WrapSubscriber(w SubscriberWrapper) Option { - return func(o *Options) { - o.SubWrappers = append(o.SubWrappers, w) - } -} - -// WrapBatchSubscriber adds a batch subscriber Wrapper to a list of options passed into the server -func WrapBatchSubscriber(w BatchSubscriberWrapper) Option { - return func(o *Options) { - o.BatchSubWrappers = append(o.BatchSubWrappers, w) - } -} - // MaxConn specifies maximum number of max simultaneous connections to server func MaxConn(n int) Option { return func(o *Options) { @@ -461,3 +435,10 @@ func SubscriberBatchWait(td time.Duration) SubscriberOption { o.BatchWait = td } } + +// Hooks sets hook runs before action +func Hooks(h ...options.Hook) Option { + return func(o *Options) { + o.Hooks = append(o.Hooks, h...) + } +} diff --git a/server/request.go b/server/request.go deleted file mode 100644 index 6b20e988..00000000 --- a/server/request.go +++ /dev/null @@ -1,35 +0,0 @@ -package server - -import ( - "go.unistack.org/micro/v3/codec" - "go.unistack.org/micro/v3/metadata" -) - -type rpcMessage struct { - payload interface{} - codec codec.Codec - header metadata.Metadata - topic string - contentType string - body []byte -} - -func (r *rpcMessage) ContentType() string { - return r.contentType -} - -func (r *rpcMessage) Topic() string { - return r.topic -} - -func (r *rpcMessage) Body() interface{} { - return r.payload -} - -func (r *rpcMessage) Header() metadata.Metadata { - return r.header -} - -func (r *rpcMessage) Codec() codec.Codec { - return r.codec -} diff --git a/server/server.go b/server/server.go index f6ea3956..eca9e82e 100644 --- a/server/server.go +++ b/server/server.go @@ -62,6 +62,13 @@ type Server interface { String() string } +type ( + FuncBatchSubHandler func(ctxs []context.Context, ms []Message) error + HookBatchSubHandler func(next FuncBatchSubHandler) FuncBatchSubHandler + FuncSubHandler func(ctx context.Context, ms Message) error + HookSubHandler func(next FuncSubHandler) FuncSubHandler +) + /* // Router handle serving messages type Router interface { @@ -147,12 +154,11 @@ type Stream interface { // // Example: // -// type Greeter struct {} -// -// func (g *Greeter) Hello(context, request, response) error { -// return nil -// } +// type Greeter struct {} // +// func (g *Greeter) Hello(context, request, response) error { +// return nil +// } type Handler interface { Name() string Handler() interface{} diff --git a/server/subscriber.go b/server/subscriber.go index 82688e4d..beed7dfe 100644 --- a/server/subscriber.go +++ b/server/subscriber.go @@ -1,21 +1,11 @@ package server import ( - "bytes" - "context" "fmt" "reflect" - "runtime/debug" "strings" "unicode" "unicode/utf8" - - "go.unistack.org/micro/v3/broker" - "go.unistack.org/micro/v3/codec" - "go.unistack.org/micro/v3/errors" - "go.unistack.org/micro/v3/logger" - "go.unistack.org/micro/v3/metadata" - "go.unistack.org/micro/v3/register" ) const ( @@ -27,26 +17,10 @@ const ( // because Typeof takes an empty interface value. This is annoying. var typeOfError = reflect.TypeOf((*error)(nil)).Elem() -type handler struct { - reqType reflect.Type - ctxType reflect.Type - method reflect.Value -} - -type subscriber struct { - typ reflect.Type - subscriber interface{} - topic string - endpoints []*register.Endpoint - handlers []*handler - opts SubscriberOptions - rcvr reflect.Value -} - // Is this an exported - upper case - name? func isExported(name string) bool { - rune, _ := utf8.DecodeRuneInString(name) - return unicode.IsUpper(rune) + r, _ := utf8.DecodeRuneInString(name) + return unicode.IsUpper(r) } // Is this type exported or a builtin? @@ -120,318 +94,3 @@ func ValidateSubscriber(sub Subscriber) error { return nil } - -func newSubscriber(topic string, sub interface{}, opts ...SubscriberOption) Subscriber { - var endpoints []*register.Endpoint - var handlers []*handler - - options := NewSubscriberOptions(opts...) - - if typ := reflect.TypeOf(sub); typ.Kind() == reflect.Func { - h := &handler{ - method: reflect.ValueOf(sub), - } - - switch typ.NumIn() { - case 1: - h.reqType = typ.In(0) - case 2: - h.ctxType = typ.In(0) - h.reqType = typ.In(1) - } - - handlers = append(handlers, h) - ep := ®ister.Endpoint{ - Name: "Func", - Request: register.ExtractSubValue(typ), - Metadata: metadata.New(2), - } - ep.Metadata.Set("topic", topic) - ep.Metadata.Set("subscriber", "true") - endpoints = append(endpoints, ep) - } else { - hdlr := reflect.ValueOf(sub) - name := reflect.Indirect(hdlr).Type().Name() - - for m := 0; m < typ.NumMethod(); m++ { - method := typ.Method(m) - h := &handler{ - method: method.Func, - } - - switch method.Type.NumIn() { - case 2: - h.reqType = method.Type.In(1) - case 3: - h.ctxType = method.Type.In(1) - h.reqType = method.Type.In(2) - } - - handlers = append(handlers, h) - ep := ®ister.Endpoint{ - Name: name + "." + method.Name, - Request: register.ExtractSubValue(method.Type), - Metadata: metadata.New(2), - } - ep.Metadata.Set("topic", topic) - ep.Metadata.Set("subscriber", "true") - endpoints = append(endpoints, ep) - } - } - - return &subscriber{ - rcvr: reflect.ValueOf(sub), - typ: reflect.TypeOf(sub), - topic: topic, - subscriber: sub, - handlers: handlers, - endpoints: endpoints, - opts: options, - } -} - -//nolint:gocyclo -func (n *noopServer) createBatchSubHandler(sb *subscriber, opts Options) broker.BatchHandler { - return func(ps broker.Events) (err error) { - defer func() { - if r := recover(); r != nil { - n.RLock() - config := n.opts - n.RUnlock() - if config.Logger.V(logger.ErrorLevel) { - config.Logger.Error(n.opts.Context, "panic recovered: ", r) - config.Logger.Error(n.opts.Context, string(debug.Stack())) - } - err = errors.InternalServerError(n.opts.Name+".subscriber", "panic recovered: %v", r) - } - }() - - msgs := make([]Message, 0, len(ps)) - ctxs := make([]context.Context, 0, len(ps)) - for _, p := range ps { - msg := p.Message() - // if we don't have headers, create empty map - if msg.Header == nil { - msg.Header = metadata.New(2) - } - - ct, _ := msg.Header.Get(metadata.HeaderContentType) - if len(ct) == 0 { - msg.Header.Set(metadata.HeaderContentType, defaultContentType) - ct = defaultContentType - } - hdr := metadata.Copy(msg.Header) - topic, _ := msg.Header.Get(metadata.HeaderTopic) - ctxs = append(ctxs, metadata.NewIncomingContext(sb.opts.Context, hdr)) - msgs = append(msgs, &rpcMessage{ - topic: topic, - contentType: ct, - header: msg.Header, - body: msg.Body, - }) - } - results := make(chan error, len(sb.handlers)) - - for i := 0; i < len(sb.handlers); i++ { - handler := sb.handlers[i] - - var req reflect.Value - - switch handler.reqType.Kind() { - case reflect.Ptr: - req = reflect.New(handler.reqType.Elem()) - default: - req = reflect.New(handler.reqType.Elem()).Elem() - } - - reqType := handler.reqType - var cf codec.Codec - for _, msg := range msgs { - cf, err = n.newCodec(msg.ContentType()) - if err != nil { - return err - } - rb := reflect.New(req.Type().Elem()) - if err = cf.ReadBody(bytes.NewReader(msg.(*rpcMessage).body), rb.Interface()); err != nil { - return err - } - msg.(*rpcMessage).codec = cf - msg.(*rpcMessage).payload = rb.Interface() - } - - fn := func(ctxs []context.Context, ms []Message) error { - var vals []reflect.Value - if sb.typ.Kind() != reflect.Func { - vals = append(vals, sb.rcvr) - } - if handler.ctxType != nil { - vals = append(vals, reflect.ValueOf(ctxs)) - } - payloads := reflect.MakeSlice(reqType, 0, len(ms)) - for _, m := range ms { - payloads = reflect.Append(payloads, reflect.ValueOf(m.Body())) - } - vals = append(vals, payloads) - - returnValues := handler.method.Call(vals) - if rerr := returnValues[0].Interface(); rerr != nil { - return rerr.(error) - } - return nil - } - - for i := len(opts.BatchSubWrappers); i > 0; i-- { - fn = opts.BatchSubWrappers[i-1](fn) - } - - if n.wg != nil { - n.wg.Add(1) - } - go func() { - if n.wg != nil { - defer n.wg.Done() - } - results <- fn(ctxs, msgs) - }() - } - - var errors []string - for i := 0; i < len(sb.handlers); i++ { - if rerr := <-results; rerr != nil { - errors = append(errors, rerr.Error()) - } - } - if len(errors) > 0 { - err = fmt.Errorf("subscriber error: %s", strings.Join(errors, "\n")) - } - return err - } -} - -//nolint:gocyclo -func (n *noopServer) createSubHandler(sb *subscriber, opts Options) broker.Handler { - return func(p broker.Event) (err error) { - defer func() { - if r := recover(); r != nil { - n.RLock() - config := n.opts - n.RUnlock() - if config.Logger.V(logger.ErrorLevel) { - config.Logger.Error(n.opts.Context, "panic recovered: ", r) - config.Logger.Error(n.opts.Context, string(debug.Stack())) - } - err = errors.InternalServerError(n.opts.Name+".subscriber", "panic recovered: %v", r) - } - }() - - msg := p.Message() - // if we don't have headers, create empty map - if msg.Header == nil { - msg.Header = metadata.New(2) - } - - ct := msg.Header["Content-Type"] - if len(ct) == 0 { - msg.Header.Set(metadata.HeaderContentType, defaultContentType) - ct = defaultContentType - } - cf, err := n.newCodec(ct) - if err != nil { - return err - } - - hdr := metadata.New(len(msg.Header)) - for k, v := range msg.Header { - hdr.Set(k, v) - } - - ctx := metadata.NewIncomingContext(sb.opts.Context, hdr) - - results := make(chan error, len(sb.handlers)) - - for i := 0; i < len(sb.handlers); i++ { - handler := sb.handlers[i] - - var isVal bool - var req reflect.Value - - if handler.reqType.Kind() == reflect.Ptr { - req = reflect.New(handler.reqType.Elem()) - } else { - req = reflect.New(handler.reqType) - isVal = true - } - if isVal { - req = req.Elem() - } - - if err = cf.ReadBody(bytes.NewBuffer(msg.Body), req.Interface()); err != nil { - return err - } - - fn := func(ctx context.Context, msg Message) error { - var vals []reflect.Value - if sb.typ.Kind() != reflect.Func { - vals = append(vals, sb.rcvr) - } - if handler.ctxType != nil { - vals = append(vals, reflect.ValueOf(ctx)) - } - - vals = append(vals, reflect.ValueOf(msg.Body())) - - returnValues := handler.method.Call(vals) - if rerr := returnValues[0].Interface(); rerr != nil { - return rerr.(error) - } - return nil - } - - for i := len(opts.SubWrappers); i > 0; i-- { - fn = opts.SubWrappers[i-1](fn) - } - - if n.wg != nil { - n.wg.Add(1) - } - go func() { - if n.wg != nil { - defer n.wg.Done() - } - cerr := fn(ctx, &rpcMessage{ - topic: sb.topic, - contentType: ct, - payload: req.Interface(), - header: msg.Header, - }) - results <- cerr - }() - } - var errors []string - for i := 0; i < len(sb.handlers); i++ { - if rerr := <-results; rerr != nil { - errors = append(errors, rerr.Error()) - } - } - if len(errors) > 0 { - err = fmt.Errorf("subscriber error: %s", strings.Join(errors, "\n")) - } - return err - } -} - -func (s *subscriber) Topic() string { - return s.topic -} - -func (s *subscriber) Subscriber() interface{} { - return s.subscriber -} - -func (s *subscriber) Endpoints() []*register.Endpoint { - return s.endpoints -} - -func (s *subscriber) Options() SubscriberOptions { - return s.opts -} diff --git a/store/memory.go b/store/memory/memory.go similarity index 59% rename from store/memory.go rename to store/memory/memory.go index 0e27ebec..7f012fab 100644 --- a/store/memory.go +++ b/store/memory/memory.go @@ -1,4 +1,4 @@ -package store +package memory import ( "context" @@ -6,13 +6,15 @@ import ( "strings" "time" - "github.com/patrickmn/go-cache" + cache "github.com/patrickmn/go-cache" + "go.unistack.org/micro/v3/options" + "go.unistack.org/micro/v3/store" ) // NewStore returns a memory store -func NewStore(opts ...Option) Store { +func NewStore(opts ...store.Option) store.Store { return &memoryStore{ - opts: NewOptions(opts...), + opts: store.NewOptions(opts...), store: cache.New(cache.NoExpiration, 5*time.Minute), } } @@ -27,8 +29,13 @@ func (m *memoryStore) Disconnect(ctx context.Context) error { } type memoryStore struct { - store *cache.Cache - opts Options + funcRead store.FuncRead + funcWrite store.FuncWrite + funcExists store.FuncExists + funcList store.FuncList + funcDelete store.FuncDelete + store *cache.Cache + opts store.Options } func (m *memoryStore) key(prefix, key string) string { @@ -39,7 +46,7 @@ func (m *memoryStore) exists(prefix, key string) error { key = m.key(prefix, key) _, found := m.store.Get(key) if !found { - return ErrNotFound + return store.ErrNotFound } return nil @@ -50,12 +57,12 @@ func (m *memoryStore) get(prefix, key string, val interface{}) error { r, found := m.store.Get(key) if !found { - return ErrNotFound + return store.ErrNotFound } buf, ok := r.([]byte) if !ok { - return ErrNotFound + return store.ErrNotFound } return m.opts.Codec.Unmarshal(buf, val) @@ -100,10 +107,32 @@ func (m *memoryStore) list(prefix string, limit, offset uint) []string { return allKeys } -func (m *memoryStore) Init(opts ...Option) error { +func (m *memoryStore) Init(opts ...store.Option) error { for _, o := range opts { o(&m.opts) } + + m.funcRead = m.fnRead + m.funcWrite = m.fnWrite + m.funcExists = m.fnExists + m.funcList = m.fnList + m.funcDelete = m.fnDelete + + m.opts.Hooks.EachNext(func(hook options.Hook) { + switch h := hook.(type) { + case store.HookRead: + m.funcRead = h(m.funcRead) + case store.HookWrite: + m.funcWrite = h(m.funcWrite) + case store.HookExists: + m.funcExists = h(m.funcExists) + case store.HookList: + m.funcList = h(m.funcList) + case store.HookDelete: + m.funcDelete = h(m.funcDelete) + } + }) + return nil } @@ -115,24 +144,36 @@ func (m *memoryStore) Name() string { return m.opts.Name } -func (m *memoryStore) Exists(ctx context.Context, key string, opts ...ExistsOption) error { - options := NewExistsOptions(opts...) +func (m *memoryStore) Exists(ctx context.Context, key string, opts ...store.ExistsOption) error { + return m.funcExists(ctx, key, opts...) +} + +func (m *memoryStore) fnExists(ctx context.Context, key string, opts ...store.ExistsOption) error { + options := store.NewExistsOptions(opts...) if options.Namespace == "" { options.Namespace = m.opts.Namespace } return m.exists(options.Namespace, key) } -func (m *memoryStore) Read(ctx context.Context, key string, val interface{}, opts ...ReadOption) error { - options := NewReadOptions(opts...) +func (m *memoryStore) Read(ctx context.Context, key string, val interface{}, opts ...store.ReadOption) error { + return m.funcRead(ctx, key, val, opts...) +} + +func (m *memoryStore) fnRead(ctx context.Context, key string, val interface{}, opts ...store.ReadOption) error { + options := store.NewReadOptions(opts...) if options.Namespace == "" { options.Namespace = m.opts.Namespace } return m.get(options.Namespace, key, val) } -func (m *memoryStore) Write(ctx context.Context, key string, val interface{}, opts ...WriteOption) error { - options := NewWriteOptions(opts...) +func (m *memoryStore) Write(ctx context.Context, key string, val interface{}, opts ...store.WriteOption) error { + return m.funcWrite(ctx, key, val, opts...) +} + +func (m *memoryStore) fnWrite(ctx context.Context, key string, val interface{}, opts ...store.WriteOption) error { + options := store.NewWriteOptions(opts...) if options.Namespace == "" { options.Namespace = m.opts.Namespace } @@ -151,8 +192,12 @@ func (m *memoryStore) Write(ctx context.Context, key string, val interface{}, op return nil } -func (m *memoryStore) Delete(ctx context.Context, key string, opts ...DeleteOption) error { - options := NewDeleteOptions(opts...) +func (m *memoryStore) Delete(ctx context.Context, key string, opts ...store.DeleteOption) error { + return m.funcDelete(ctx, key, opts...) +} + +func (m *memoryStore) fnDelete(ctx context.Context, key string, opts ...store.DeleteOption) error { + options := store.NewDeleteOptions(opts...) if options.Namespace == "" { options.Namespace = m.opts.Namespace } @@ -161,12 +206,16 @@ func (m *memoryStore) Delete(ctx context.Context, key string, opts ...DeleteOpti return nil } -func (m *memoryStore) Options() Options { +func (m *memoryStore) Options() store.Options { return m.opts } -func (m *memoryStore) List(ctx context.Context, opts ...ListOption) ([]string, error) { - options := NewListOptions(opts...) +func (m *memoryStore) List(ctx context.Context, opts ...store.ListOption) ([]string, error) { + return m.funcList(ctx, opts...) +} + +func (m *memoryStore) fnList(ctx context.Context, opts ...store.ListOption) ([]string, error) { + options := store.NewListOptions(opts...) if options.Namespace == "" { options.Namespace = m.opts.Namespace } diff --git a/store/memory_test.go b/store/memory/memory_test.go similarity index 66% rename from store/memory_test.go rename to store/memory/memory_test.go index da8a9ad0..598f00eb 100644 --- a/store/memory_test.go +++ b/store/memory/memory_test.go @@ -1,4 +1,4 @@ -package store_test +package memory import ( "context" @@ -8,8 +8,41 @@ import ( "go.unistack.org/micro/v3/store" ) +type testHook struct { + f bool +} + +func (t *testHook) Exists(fn store.FuncExists) store.FuncExists { + return func(ctx context.Context, key string, opts ...store.ExistsOption) error { + t.f = true + return fn(ctx, key, opts...) + } +} + +func TestHook(t *testing.T) { + h := &testHook{} + + s := NewStore(store.Hooks(store.HookExists(h.Exists))) + + if err := s.Init(); err != nil { + t.Fatal(err) + } + + if err := s.Write(context.TODO(), "test", nil); err != nil { + t.Error(err) + } + + if err := s.Exists(context.TODO(), "test"); err != nil { + t.Fatal(err) + } + + if !h.f { + t.Fatal("hook not works") + } +} + func TestMemoryReInit(t *testing.T) { - s := store.NewStore(store.Namespace("aaa")) + s := NewStore(store.Namespace("aaa")) if err := s.Init(store.Namespace("")); err != nil { t.Fatal(err) } @@ -19,7 +52,7 @@ func TestMemoryReInit(t *testing.T) { } func TestMemoryBasic(t *testing.T) { - s := store.NewStore() + s := NewStore() if err := s.Init(); err != nil { t.Fatal(err) } @@ -27,7 +60,7 @@ func TestMemoryBasic(t *testing.T) { } func TestMemoryPrefix(t *testing.T) { - s := store.NewStore() + s := NewStore() if err := s.Init(store.Namespace("some-prefix")); err != nil { t.Fatal(err) } @@ -35,7 +68,7 @@ func TestMemoryPrefix(t *testing.T) { } func TestMemoryNamespace(t *testing.T) { - s := store.NewStore() + s := NewStore() if err := s.Init(store.Namespace("some-namespace")); err != nil { t.Fatal(err) } @@ -43,7 +76,7 @@ func TestMemoryNamespace(t *testing.T) { } func TestMemoryNamespacePrefix(t *testing.T) { - s := store.NewStore() + s := NewStore() if err := s.Init(store.Namespace("some-namespace")); err != nil { t.Fatal(err) } diff --git a/store/noop.go b/store/noop.go new file mode 100644 index 00000000..c7e2922d --- /dev/null +++ b/store/noop.go @@ -0,0 +1,147 @@ +package store + +import ( + "context" + + "go.unistack.org/micro/v3/options" +) + +var _ Store = (*noopStore)(nil) + +type noopStore struct { + funcRead FuncRead + funcWrite FuncWrite + funcExists FuncExists + funcList FuncList + funcDelete FuncDelete + opts Options +} + +func NewStore(opts ...Option) *noopStore { + options := NewOptions(opts...) + return &noopStore{opts: options} +} + +func (n *noopStore) Init(opts ...Option) error { + for _, o := range opts { + o(&n.opts) + } + + n.funcRead = n.fnRead + n.funcWrite = n.fnWrite + n.funcExists = n.fnExists + n.funcList = n.fnList + n.funcDelete = n.fnDelete + + n.opts.Hooks.EachNext(func(hook options.Hook) { + switch h := hook.(type) { + case HookRead: + n.funcRead = h(n.funcRead) + case HookWrite: + n.funcWrite = h(n.funcWrite) + case HookExists: + n.funcExists = h(n.funcExists) + case HookList: + n.funcList = h(n.funcList) + case HookDelete: + n.funcDelete = h(n.funcDelete) + } + }) + + return nil +} + +func (n *noopStore) Connect(ctx context.Context) error { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + return nil +} + +func (n *noopStore) Disconnect(ctx context.Context) error { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + return nil +} + +func (n *noopStore) Read(ctx context.Context, key string, val interface{}, opts ...ReadOption) error { + return n.funcRead(ctx, key, val, opts...) +} + +func (n *noopStore) fnRead(ctx context.Context, key string, val interface{}, opts ...ReadOption) error { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + return nil +} + +func (n *noopStore) Delete(ctx context.Context, key string, opts ...DeleteOption) error { + return n.funcDelete(ctx, key, opts...) +} + +func (n *noopStore) fnDelete(ctx context.Context, key string, opts ...DeleteOption) error { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + return nil +} + +func (n *noopStore) Exists(ctx context.Context, key string, opts ...ExistsOption) error { + return n.funcExists(ctx, key, opts...) +} + +func (n *noopStore) fnExists(ctx context.Context, key string, opts ...ExistsOption) error { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + return nil +} + +func (n *noopStore) Write(ctx context.Context, key string, val interface{}, opts ...WriteOption) error { + return n.funcWrite(ctx, key, val, opts...) +} + +func (n *noopStore) fnWrite(ctx context.Context, key string, val interface{}, opts ...WriteOption) error { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + return nil +} + +func (n *noopStore) List(ctx context.Context, opts ...ListOption) ([]string, error) { + return n.funcList(ctx, opts...) +} + +func (n *noopStore) fnList(ctx context.Context, opts ...ListOption) ([]string, error) { + select { + case <-ctx.Done(): + return nil, ctx.Err() + default: + } + return nil, nil +} + +func (n *noopStore) Name() string { + return n.opts.Name +} + +func (n *noopStore) String() string { + return "noop" +} + +func (n *noopStore) Options() Options { + return n.opts +} diff --git a/store/noop_test.go b/store/noop_test.go new file mode 100644 index 00000000..19fab2d7 --- /dev/null +++ b/store/noop_test.go @@ -0,0 +1,35 @@ +package store + +import ( + "context" + "testing" +) + +type testHook struct { + f bool +} + +func (t *testHook) Exists(fn FuncExists) FuncExists { + return func(ctx context.Context, key string, opts ...ExistsOption) error { + t.f = true + return fn(ctx, key, opts...) + } +} + +func TestHook(t *testing.T) { + h := &testHook{} + + s := NewStore(Hooks(HookExists(h.Exists))) + + if err := s.Init(); err != nil { + t.Fatal(err) + } + + if err := s.Exists(context.TODO(), "test"); err != nil { + t.Fatal(err) + } + + if !h.f { + t.Fatal("hook not works") + } +} diff --git a/store/options.go b/store/options.go index 6bbee1a4..438b745c 100644 --- a/store/options.go +++ b/store/options.go @@ -9,6 +9,7 @@ import ( "go.unistack.org/micro/v3/logger" "go.unistack.org/micro/v3/metadata" "go.unistack.org/micro/v3/meter" + "go.unistack.org/micro/v3/options" "go.unistack.org/micro/v3/tracer" ) @@ -38,6 +39,8 @@ type Options struct { // Wrappers []Wrapper // Timeout specifies timeout duration for all operations Timeout time.Duration + // Hooks can be run before/after store Read/List/Write/Exists/Delete + Hooks options.Hooks } // NewOptions creates options struct @@ -441,11 +444,9 @@ func ExistsTimeout(td time.Duration) ExistsOption { } } -/* -// WrapStore adds a store Wrapper to a list of options passed into the store -func WrapStore(w Wrapper) Option { +// Hooks sets hook runs before action +func Hooks(h ...options.Hook) Option { return func(o *Options) { - o.Wrappers = append(o.Wrappers, w) + o.Hooks = append(o.Hooks, h...) } } -*/ diff --git a/store/store.go b/store/store.go index 896c1350..57646346 100644 --- a/store/store.go +++ b/store/store.go @@ -19,6 +19,7 @@ var ( // Store is a data storage interface type Store interface { + // Name returns store name Name() string // Init initialises the store Init(opts ...Option) error @@ -41,3 +42,16 @@ type Store interface { // String returns the name of the implementation. String() string } + +type ( + FuncExists func(ctx context.Context, key string, opts ...ExistsOption) error + HookExists func(next FuncExists) FuncExists + FuncRead func(ctx context.Context, key string, val interface{}, opts ...ReadOption) error + HookRead func(next FuncRead) FuncRead + FuncWrite func(ctx context.Context, key string, val interface{}, opts ...WriteOption) error + HookWrite func(next FuncWrite) FuncWrite + FuncDelete func(ctx context.Context, key string, opts ...DeleteOption) error + HookDelete func(next FuncDelete) FuncDelete + FuncList func(ctx context.Context, opts ...ListOption) ([]string, error) + HookList func(next FuncList) FuncList +) From 3a21069b864ba31c4a92bfefe115c2455ebcae3f Mon Sep 17 00:00:00 2001 From: Vasiliy Tolstov Date: Mon, 22 Apr 2024 08:50:33 +0300 Subject: [PATCH 12/59] remote stale test Signed-off-by: Vasiliy Tolstov --- tracer/tracer_test.go | 13 ------------- 1 file changed, 13 deletions(-) delete mode 100644 tracer/tracer_test.go diff --git a/tracer/tracer_test.go b/tracer/tracer_test.go deleted file mode 100644 index 7388f577..00000000 --- a/tracer/tracer_test.go +++ /dev/null @@ -1,13 +0,0 @@ -package tracer - -import ( - "testing" -) - -func TestUniqLabels(t *testing.T) { - labels := []interface{}{"key1", "val1", "key1", "val2"} - labels = UniqLabels(labels) - if labels[1] != "val2" { - t.Fatalf("UniqLabels not works") - } -} From 781dee03db38ecd38ed14acebabcbeaafb68ae29 Mon Sep 17 00:00:00 2001 From: Vasiliy Tolstov Date: Mon, 22 Apr 2024 23:39:21 +0300 Subject: [PATCH 13/59] add client tracing Signed-off-by: Vasiliy Tolstov --- client/noop.go | 70 ++++++++++++++++++++++++++++++++++++++++++------ errors/errors.go | 4 +++ 2 files changed, 66 insertions(+), 8 deletions(-) diff --git a/client/noop.go b/client/noop.go index b5ae7791..5fb8d571 100644 --- a/client/noop.go +++ b/client/noop.go @@ -4,6 +4,7 @@ import ( "context" "fmt" "os" + "strconv" "time" "go.unistack.org/micro/v3/broker" @@ -12,6 +13,8 @@ import ( "go.unistack.org/micro/v3/metadata" "go.unistack.org/micro/v3/options" "go.unistack.org/micro/v3/selector" + "go.unistack.org/micro/v3/semconv" + "go.unistack.org/micro/v3/tracer" ) // DefaultCodecs will be used to encode/decode data @@ -104,10 +107,13 @@ func (n *noopResponse) Read() ([]byte, error) { return nil, nil } -type noopStream struct{} +type noopStream struct { + err error + ctx context.Context +} func (n *noopStream) Context() context.Context { - return context.Background() + return n.ctx } func (n *noopStream) Request() Request { @@ -135,15 +141,21 @@ func (n *noopStream) RecvMsg(interface{}) error { } func (n *noopStream) Error() error { - return nil + return n.err } func (n *noopStream) Close() error { - return nil + if sp, ok := tracer.SpanFromContext(n.ctx); ok && sp != nil { + if n.err != nil { + sp.SetStatus(tracer.SpanStatusError, n.err.Error()) + } + sp.Finish() + } + return n.err } func (n *noopStream) CloseSend() error { - return nil + return n.err } func (n *noopMessage) Topic() string { @@ -207,7 +219,28 @@ func (n *noopClient) String() string { } func (n *noopClient) Call(ctx context.Context, req Request, rsp interface{}, opts ...CallOption) error { - return n.funcCall(ctx, req, rsp, opts...) + ts := time.Now() + n.opts.Meter.Counter(semconv.ClientRequestInflight, "endpoint", req.Endpoint()).Inc() + var sp tracer.Span + ctx, sp = n.opts.Tracer.Start(ctx, req.Endpoint()+" rpc-client", + tracer.WithSpanKind(tracer.SpanKindClient), + tracer.WithSpanLabels("endpoint", req.Endpoint()), + ) + err := n.funcCall(ctx, req, rsp, opts...) + n.opts.Meter.Counter(semconv.ClientRequestInflight, "endpoint", req.Endpoint()).Dec() + te := time.Since(ts) + n.opts.Meter.Summary(semconv.ClientRequestLatencyMicroseconds, "endpoint", req.Endpoint()).Update(te.Seconds()) + n.opts.Meter.Histogram(semconv.ClientRequestDurationSeconds, "endpoint", req.Endpoint()).Update(te.Seconds()) + + if me := errors.FromError(err); me == nil { + sp.Finish() + n.opts.Meter.Counter(semconv.ClientRequestTotal, "endpoint", req.Endpoint(), "status", "success", "code", strconv.Itoa(int(200))).Inc() + } else { + sp.SetStatus(tracer.SpanStatusError, err.Error()) + n.opts.Meter.Counter(semconv.ClientRequestTotal, "endpoint", req.Endpoint(), "status", "failure", "code", strconv.Itoa(int(me.Code))).Inc() + } + + return err } func (n *noopClient) fnCall(ctx context.Context, req Request, rsp interface{}, opts ...CallOption) error { @@ -349,7 +382,28 @@ func (n *noopClient) NewMessage(topic string, msg interface{}, opts ...MessageOp } func (n *noopClient) Stream(ctx context.Context, req Request, opts ...CallOption) (Stream, error) { - return n.funcStream(ctx, req, opts...) + ts := time.Now() + n.opts.Meter.Counter(semconv.ClientRequestInflight, "endpoint", req.Endpoint()).Inc() + var sp tracer.Span + ctx, sp = n.opts.Tracer.Start(ctx, req.Endpoint()+" rpc-client", + tracer.WithSpanKind(tracer.SpanKindClient), + tracer.WithSpanLabels("endpoint", req.Endpoint()), + ) + stream, err := n.funcStream(ctx, req, opts...) + n.opts.Meter.Counter(semconv.ClientRequestInflight, "endpoint", req.Endpoint()).Dec() + te := time.Since(ts) + n.opts.Meter.Summary(semconv.ClientRequestLatencyMicroseconds, "endpoint", req.Endpoint()).Update(te.Seconds()) + n.opts.Meter.Histogram(semconv.ClientRequestDurationSeconds, "endpoint", req.Endpoint()).Update(te.Seconds()) + + if me := errors.FromError(err); me == nil { + sp.Finish() + n.opts.Meter.Counter(semconv.ClientRequestTotal, "endpoint", req.Endpoint(), "status", "success", "code", strconv.Itoa(int(200))).Inc() + } else { + sp.SetStatus(tracer.SpanStatusError, err.Error()) + n.opts.Meter.Counter(semconv.ClientRequestTotal, "endpoint", req.Endpoint(), "status", "failure", "code", strconv.Itoa(int(me.Code))).Inc() + } + + return stream, err } func (n *noopClient) fnStream(ctx context.Context, req Request, opts ...CallOption) (Stream, error) { @@ -493,7 +547,7 @@ func (n *noopClient) fnStream(ctx context.Context, req Request, opts ...CallOpti } func (n *noopClient) stream(ctx context.Context, addr string, req Request, opts CallOptions) (Stream, error) { - return &noopStream{}, nil + return &noopStream{ctx: ctx}, nil } func (n *noopClient) BatchPublish(ctx context.Context, ps []Message, opts ...PublishOption) error { diff --git a/errors/errors.go b/errors/errors.go index e9f0e177..b59034e6 100644 --- a/errors/errors.go +++ b/errors/errors.go @@ -262,6 +262,10 @@ func CodeIn(err interface{}, codes ...int32) bool { // FromError try to convert go error to *Error func FromError(err error) *Error { + if err == nil { + return nil + } + if verr, ok := err.(*Error); ok && verr != nil { return verr } From 849c46203759d6f06d709782ed4f8e5c40178f76 Mon Sep 17 00:00:00 2001 From: Vasiliy Tolstov Date: Tue, 23 Apr 2024 07:28:58 +0300 Subject: [PATCH 14/59] server/noop: cleanup Signed-off-by: Vasiliy Tolstov --- server/noop.go | 164 +++++-------------------------------------- server/noop_test.go | 20 ------ server/options.go | 9 --- server/server.go | 6 +- server/subscriber.go | 26 +++---- server/wrapper.go | 8 --- 6 files changed, 28 insertions(+), 205 deletions(-) diff --git a/server/noop.go b/server/noop.go index f2bc031f..85333a50 100644 --- a/server/noop.go +++ b/server/noop.go @@ -274,7 +274,7 @@ func (n *noopServer) Register() error { if !registered { if config.Logger.V(logger.InfoLevel) { - config.Logger.Infof(n.opts.Context, "register [%s] Registering node: %s", config.Register.String(), service.Nodes[0].ID) + config.Logger.Info(n.opts.Context, fmt.Sprintf("register [%s] Registering node: %s", config.Register.String(), service.Nodes[0].ID)) } } @@ -312,7 +312,7 @@ func (n *noopServer) Deregister() error { } if config.Logger.V(logger.InfoLevel) { - config.Logger.Infof(n.opts.Context, "deregistering node: %s", service.Nodes[0].ID) + config.Logger.Info(n.opts.Context, fmt.Sprintf("deregistering node: %s", service.Nodes[0].ID)) } if err := DefaultDeregisterFunc(service, config); err != nil { @@ -343,11 +343,11 @@ func (n *noopServer) Deregister() error { go func(s broker.Subscriber) { defer wg.Done() if config.Logger.V(logger.InfoLevel) { - config.Logger.Infof(n.opts.Context, "unsubscribing from topic: %s", s.Topic()) + config.Logger.Info(n.opts.Context, "unsubscribing from topic: "+s.Topic()) } if err := s.Unsubscribe(ncx); err != nil { if config.Logger.V(logger.ErrorLevel) { - config.Logger.Errorf(n.opts.Context, "unsubscribing from topic: %s err: %v", s.Topic(), err) + config.Logger.Error(n.opts.Context, "unsubscribing from topic: "+s.Topic(), err) } } }(subs[idx]) @@ -383,7 +383,7 @@ func (n *noopServer) Start() error { config.Address = addr if config.Logger.V(logger.InfoLevel) { - config.Logger.Infof(n.opts.Context, "server [noop] Listening on %s", config.Address) + config.Logger.Info(n.opts.Context, "server [noop] Listening on "+config.Address) } n.Lock() @@ -397,13 +397,13 @@ func (n *noopServer) Start() error { // connect to the broker if err := config.Broker.Connect(config.Context); err != nil { if config.Logger.V(logger.ErrorLevel) { - config.Logger.Errorf(n.opts.Context, "broker [%s] connect error: %v", config.Broker.String(), err) + config.Logger.Error(n.opts.Context, fmt.Sprintf("broker [%s] connect error", config.Broker.String()), err) } return err } if config.Logger.V(logger.InfoLevel) { - config.Logger.Infof(n.opts.Context, "broker [%s] Connected to %s", config.Broker.String(), config.Broker.Address()) + config.Logger.Info(n.opts.Context, fmt.Sprintf("broker [%s] Connected to %s", config.Broker.String(), config.Broker.Address())) } } @@ -411,13 +411,13 @@ func (n *noopServer) Start() error { // nolint: nestif if err := config.RegisterCheck(config.Context); err != nil { if config.Logger.V(logger.ErrorLevel) { - config.Logger.Errorf(n.opts.Context, "server %s-%s register check error: %s", config.Name, config.ID, err) + config.Logger.Error(n.opts.Context, fmt.Sprintf("server %s-%s register check error", config.Name, config.ID), err) } } else { // announce self to the world if err := n.Register(); err != nil { if config.Logger.V(logger.ErrorLevel) { - config.Logger.Errorf(n.opts.Context, "server register error: %v", err) + config.Logger.Error(n.opts.Context, "server register error", err) } } } @@ -450,23 +450,23 @@ func (n *noopServer) Start() error { // nolint: nestif if rerr != nil && registered { if config.Logger.V(logger.ErrorLevel) { - config.Logger.Errorf(n.opts.Context, "server %s-%s register check error: %s, deregister it", config.Name, config.ID, rerr) + config.Logger.Error(n.opts.Context, fmt.Sprintf("server %s-%s register check error, deregister it", config.Name, config.ID), rerr) } // deregister self in case of error if err := n.Deregister(); err != nil { if config.Logger.V(logger.ErrorLevel) { - config.Logger.Errorf(n.opts.Context, "server %s-%s deregister error: %s", config.Name, config.ID, err) + config.Logger.Error(n.opts.Context, fmt.Sprintf("server %s-%s deregister error", config.Name, config.ID), err) } } } else if rerr != nil && !registered { if config.Logger.V(logger.ErrorLevel) { - config.Logger.Errorf(n.opts.Context, "server %s-%s register check error: %s", config.Name, config.ID, rerr) + config.Logger.Errorf(n.opts.Context, fmt.Sprintf("server %s-%s register check error", config.Name, config.ID), rerr) } continue } if err := n.Register(); err != nil { if config.Logger.V(logger.ErrorLevel) { - config.Logger.Errorf(n.opts.Context, "server %s-%s register error: %s", config.Name, config.ID, err) + config.Logger.Error(n.opts.Context, fmt.Sprintf("server %s-%s register error", config.Name, config.ID), err) } } // wait for exit @@ -478,7 +478,7 @@ func (n *noopServer) Start() error { // deregister self if err := n.Deregister(); err != nil { if config.Logger.V(logger.ErrorLevel) { - config.Logger.Errorf(n.opts.Context, "server deregister error: ", err) + config.Logger.Error(n.opts.Context, "server deregister error", err) } } @@ -491,12 +491,12 @@ func (n *noopServer) Start() error { ch <- nil if config.Logger.V(logger.InfoLevel) { - config.Logger.Infof(n.opts.Context, "broker [%s] Disconnected from %s", config.Broker.String(), config.Broker.Address()) + config.Logger.Info(n.opts.Context, fmt.Sprintf("broker [%s] Disconnected from %s", config.Broker.String(), config.Broker.Address())) } // disconnect broker if err := config.Broker.Disconnect(config.Context); err != nil { if config.Logger.V(logger.ErrorLevel) { - config.Logger.Errorf(n.opts.Context, "broker [%s] disconnect error: %v", config.Broker.String(), err) + config.Logger.Error(n.opts.Context, fmt.Sprintf("broker [%s] disconnect error", config.Broker.String()), err) } } }() @@ -526,20 +526,13 @@ func (n *noopServer) subscribe() error { opts = append(opts, broker.SubscribeGroup(queue)) } - if sb.Options().Batch { - // batch processing handler - sub, err = config.Broker.BatchSubscribe(cx, sb.Topic(), n.createBatchSubHandler(sb, config), opts...) - } else { - // single processing handler - sub, err = config.Broker.Subscribe(cx, sb.Topic(), n.createSubHandler(sb, config), opts...) - } - + sub, err = config.Broker.Subscribe(cx, sb.Topic(), n.createSubHandler(sb, config), opts...) if err != nil { return err } if config.Logger.V(logger.InfoLevel) { - config.Logger.Infof(n.opts.Context, "subscribing to topic: %s", sb.Topic()) + config.Logger.Info(n.opts.Context, "subscribing to topic: "+sb.Topic()) } n.subscribers[sb] = []broker.Subscriber{sub} @@ -637,127 +630,6 @@ func newSubscriber(topic string, sub interface{}, opts ...SubscriberOption) Subs } } -//nolint:gocyclo -func (n *noopServer) createBatchSubHandler(sb *subscriber, opts Options) broker.BatchHandler { - return func(ps broker.Events) (err error) { - defer func() { - if r := recover(); r != nil { - n.RLock() - config := n.opts - n.RUnlock() - if config.Logger.V(logger.ErrorLevel) { - config.Logger.Error(n.opts.Context, "panic recovered: ", r) - config.Logger.Error(n.opts.Context, string(debug.Stack())) - } - err = errors.InternalServerError(n.opts.Name+".subscriber", "panic recovered: %v", r) - } - }() - - msgs := make([]Message, 0, len(ps)) - ctxs := make([]context.Context, 0, len(ps)) - for _, p := range ps { - msg := p.Message() - // if we don't have headers, create empty map - if msg.Header == nil { - msg.Header = metadata.New(2) - } - - ct, _ := msg.Header.Get(metadata.HeaderContentType) - if len(ct) == 0 { - msg.Header.Set(metadata.HeaderContentType, defaultContentType) - ct = defaultContentType - } - hdr := metadata.Copy(msg.Header) - topic, _ := msg.Header.Get(metadata.HeaderTopic) - ctxs = append(ctxs, metadata.NewIncomingContext(sb.opts.Context, hdr)) - msgs = append(msgs, &rpcMessage{ - topic: topic, - contentType: ct, - header: msg.Header, - body: msg.Body, - }) - } - results := make(chan error, len(sb.handlers)) - - for i := 0; i < len(sb.handlers); i++ { - handler := sb.handlers[i] - - var req reflect.Value - - switch handler.reqType.Kind() { - case reflect.Ptr: - req = reflect.New(handler.reqType.Elem()) - default: - req = reflect.New(handler.reqType.Elem()).Elem() - } - - reqType := handler.reqType - var cf codec.Codec - for _, msg := range msgs { - cf, err = n.newCodec(msg.ContentType()) - if err != nil { - return err - } - rb := reflect.New(req.Type().Elem()) - if err = cf.ReadBody(bytes.NewReader(msg.(*rpcMessage).body), rb.Interface()); err != nil { - return err - } - msg.(*rpcMessage).codec = cf - msg.(*rpcMessage).payload = rb.Interface() - } - - fn := func(ctxs []context.Context, ms []Message) error { - var vals []reflect.Value - if sb.typ.Kind() != reflect.Func { - vals = append(vals, sb.rcvr) - } - if handler.ctxType != nil { - vals = append(vals, reflect.ValueOf(ctxs)) - } - payloads := reflect.MakeSlice(reqType, 0, len(ms)) - for _, m := range ms { - payloads = reflect.Append(payloads, reflect.ValueOf(m.Body())) - } - vals = append(vals, payloads) - - returnValues := handler.method.Call(vals) - if rerr := returnValues[0].Interface(); rerr != nil { - return rerr.(error) - } - return nil - } - - opts.Hooks.EachNext(func(hook options.Hook) { - if h, ok := hook.(HookBatchSubHandler); ok { - fn = h(fn) - } - }) - - if n.wg != nil { - n.wg.Add(1) - } - - go func() { - if n.wg != nil { - defer n.wg.Done() - } - results <- fn(ctxs, msgs) - }() - } - - var errors []string - for i := 0; i < len(sb.handlers); i++ { - if rerr := <-results; rerr != nil { - errors = append(errors, rerr.Error()) - } - } - if len(errors) > 0 { - err = fmt.Errorf("subscriber error: %s", strings.Join(errors, "\n")) - } - return err - } -} - //nolint:gocyclo func (n *noopServer) createSubHandler(sb *subscriber, opts Options) broker.Handler { return func(p broker.Event) (err error) { diff --git a/server/noop_test.go b/server/noop_test.go index 5ecc762d..2bad82e6 100644 --- a/server/noop_test.go +++ b/server/noop_test.go @@ -9,7 +9,6 @@ import ( "go.unistack.org/micro/v3/client" "go.unistack.org/micro/v3/codec" "go.unistack.org/micro/v3/logger" - "go.unistack.org/micro/v3/metadata" "go.unistack.org/micro/v3/server" ) @@ -26,18 +25,6 @@ func (h *TestHandler) SingleSubHandler(ctx context.Context, msg *codec.Frame) er return nil } -func (h *TestHandler) BatchSubHandler(ctxs []context.Context, msgs []*codec.Frame) error { - if len(msgs) != 8 { - h.t.Fatal("invalid number of messages received") - } - for idx := 0; idx < len(msgs); idx++ { - md, _ := metadata.FromIncomingContext(ctxs[idx]) - _ = md - // fmt.Printf("msg md %v\n", md) - } - return nil -} - func TestNoopSub(t *testing.T) { ctx := context.Background() @@ -76,13 +63,6 @@ func TestNoopSub(t *testing.T) { t.Fatal(err) } - if err := s.Subscribe(s.NewSubscriber("batch_topic", h.BatchSubHandler, - server.SubscriberQueue("queue"), - server.SubscriberBatch(true), - )); err != nil { - t.Fatal(err) - } - if err := s.Start(); err != nil { t.Fatal(err) } diff --git a/server/options.go b/server/options.go index 093b5e0e..ffced1a4 100644 --- a/server/options.go +++ b/server/options.go @@ -341,8 +341,6 @@ type SubscriberOptions struct { AutoAck bool // BodyOnly flag specifies that message without headers BodyOnly bool - // Batch flag specifies that message processed in batches - Batch bool // BatchSize flag specifies max size of batch BatchSize int // BatchWait flag specifies max wait time for batch filling @@ -414,13 +412,6 @@ func SubscriberAck(b bool) SubscriberOption { } } -// SubscriberBatch control batch processing for handler -func SubscriberBatch(b bool) SubscriberOption { - return func(o *SubscriberOptions) { - o.Batch = b - } -} - // SubscriberBatchSize control batch filling size for handler // Batch filling max waiting time controlled by SubscriberBatchWait func SubscriberBatchSize(n int) SubscriberOption { diff --git a/server/server.go b/server/server.go index eca9e82e..3879812e 100644 --- a/server/server.go +++ b/server/server.go @@ -63,10 +63,8 @@ type Server interface { } type ( - FuncBatchSubHandler func(ctxs []context.Context, ms []Message) error - HookBatchSubHandler func(next FuncBatchSubHandler) FuncBatchSubHandler - FuncSubHandler func(ctx context.Context, ms Message) error - HookSubHandler func(next FuncSubHandler) FuncSubHandler + FuncSubHandler func(ctx context.Context, ms Message) error + HookSubHandler func(next FuncSubHandler) FuncSubHandler ) /* diff --git a/server/subscriber.go b/server/subscriber.go index beed7dfe..81ee6030 100644 --- a/server/subscriber.go +++ b/server/subscriber.go @@ -3,14 +3,12 @@ package server import ( "fmt" "reflect" - "strings" "unicode" "unicode/utf8" ) const ( - subSig = "func(context.Context, interface{}) error" - batchSubSig = "func([]context.Context, []interface{}) error" + subSig = "func(context.Context, interface{}) error" ) // Precompute the reflect type for error. Can't use error directly @@ -43,23 +41,15 @@ func ValidateSubscriber(sub Subscriber) error { switch typ.NumIn() { case 2: argType = typ.In(1) - if sub.Options().Batch { - if argType.Kind() != reflect.Slice { - return fmt.Errorf("subscriber %v dont have required signature %s", name, batchSubSig) - } - if strings.Compare(fmt.Sprintf("%v", argType), "[]interface{}") == 0 { - return fmt.Errorf("subscriber %v dont have required signaure %s", name, batchSubSig) - } - } default: - return fmt.Errorf("subscriber %v takes wrong number of args: %v required signature %s or %s", name, typ.NumIn(), subSig, batchSubSig) + return fmt.Errorf("subscriber %v takes wrong number of args: %v required signature %s", name, typ.NumIn(), subSig) } if !isExportedOrBuiltinType(argType) { return fmt.Errorf("subscriber %v argument type not exported: %v", name, argType) } if typ.NumOut() != 1 { - return fmt.Errorf("subscriber %v has wrong number of return values: %v require signature %s or %s", - name, typ.NumOut(), subSig, batchSubSig) + return fmt.Errorf("subscriber %v has wrong number of return values: %v require signature %s", + name, typ.NumOut(), subSig) } if returnType := typ.Out(0); returnType != typeOfError { return fmt.Errorf("subscriber %v returns %v not error", name, returnType.String()) @@ -74,8 +64,8 @@ func ValidateSubscriber(sub Subscriber) error { case 3: argType = method.Type.In(2) default: - return fmt.Errorf("subscriber %v.%v takes wrong number of args: %v required signature %s or %s", - name, method.Name, method.Type.NumIn(), subSig, batchSubSig) + return fmt.Errorf("subscriber %v.%v takes wrong number of args: %v required signature %s", + name, method.Name, method.Type.NumIn(), subSig) } if !isExportedOrBuiltinType(argType) { @@ -83,8 +73,8 @@ func ValidateSubscriber(sub Subscriber) error { } if method.Type.NumOut() != 1 { return fmt.Errorf( - "subscriber %v.%v has wrong number of return values: %v require signature %s or %s", - name, method.Name, method.Type.NumOut(), subSig, batchSubSig) + "subscriber %v.%v has wrong number of return values: %v require signature %s", + name, method.Name, method.Type.NumOut(), subSig) } if returnType := method.Type.Out(0); returnType != typeOfError { return fmt.Errorf("subscriber %v.%v returns %v not error", name, method.Name, returnType.String()) diff --git a/server/wrapper.go b/server/wrapper.go index b4596e63..3e4d3ecd 100644 --- a/server/wrapper.go +++ b/server/wrapper.go @@ -14,20 +14,12 @@ type HandlerFunc func(ctx context.Context, req Request, rsp interface{}) error // publication message. type SubscriberFunc func(ctx context.Context, msg Message) error -// BatchSubscriberFunc represents a single method of a subscriber. It's used primarily -// for the wrappers. What's handed to the actual method is the concrete -// publication message. This func used by batch subscribers -type BatchSubscriberFunc func(ctxs []context.Context, msgs []Message) error - // HandlerWrapper wraps the HandlerFunc and returns the equivalent type HandlerWrapper func(HandlerFunc) HandlerFunc // SubscriberWrapper wraps the SubscriberFunc and returns the equivalent type SubscriberWrapper func(SubscriberFunc) SubscriberFunc -// BatchSubscriberWrapper wraps the SubscriberFunc and returns the equivalent -type BatchSubscriberWrapper func(BatchSubscriberFunc) BatchSubscriberFunc - // StreamWrapper wraps a Stream interface and returns the equivalent. // Because streams exist for the lifetime of a method invocation this // is a convenient way to wrap a Stream as its in use for trace, monitoring, From 4e99680c30ce9149032f204fb22c9053dd5ba0bc Mon Sep 17 00:00:00 2001 From: Vasiliy Tolstov Date: Tue, 23 Apr 2024 07:39:08 +0300 Subject: [PATCH 15/59] server: add missing hook definitions Signed-off-by: Vasiliy Tolstov --- server/noop.go | 26 +++++++++++++++----------- server/server.go | 2 ++ 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/server/noop.go b/server/noop.go index 85333a50..956ac9c2 100644 --- a/server/noop.go +++ b/server/noop.go @@ -512,29 +512,33 @@ func (n *noopServer) Start() error { func (n *noopServer) subscribe() error { config := n.Options() - cx := config.Context - var err error - var sub broker.Subscriber + subCtx := config.Context for sb := range n.subscribers { - if sb.Options().Context != nil { - cx = sb.Options().Context + + if cx := sb.Options().Context; cx != nil { + subCtx = cx + } + + opts := []broker.SubscribeOption{ + broker.SubscribeContext(subCtx), + broker.SubscribeAutoAck(sb.Options().AutoAck), + broker.SubscribeBodyOnly(sb.Options().BodyOnly), } - opts := []broker.SubscribeOption{broker.SubscribeContext(cx), broker.SubscribeAutoAck(sb.Options().AutoAck)} if queue := sb.Options().Queue; len(queue) > 0 { opts = append(opts, broker.SubscribeGroup(queue)) } - sub, err = config.Broker.Subscribe(cx, sb.Topic(), n.createSubHandler(sb, config), opts...) - if err != nil { - return err - } - if config.Logger.V(logger.InfoLevel) { config.Logger.Info(n.opts.Context, "subscribing to topic: "+sb.Topic()) } + sub, err := config.Broker.Subscribe(subCtx, sb.Topic(), n.createSubHandler(sb, config), opts...) + if err != nil { + return err + } + n.subscribers[sb] = []broker.Subscriber{sub} } diff --git a/server/server.go b/server/server.go index 3879812e..b6ada16a 100644 --- a/server/server.go +++ b/server/server.go @@ -65,6 +65,8 @@ type Server interface { type ( FuncSubHandler func(ctx context.Context, ms Message) error HookSubHandler func(next FuncSubHandler) FuncSubHandler + FuncHandler func(ctx context.Context, req Request, rsp interface{}) error + HookHandler func(next FuncHandler) FuncHandler ) /* From 1a1459dd0ec5bc17a3e76fb8abac36e4b31da932 Mon Sep 17 00:00:00 2001 From: Vasiliy Tolstov Date: Sat, 4 May 2024 13:16:31 +0300 Subject: [PATCH 16/59] util/reflect: fix StructFieldByTag Signed-off-by: Vasiliy Tolstov --- util/reflect/struct.go | 18 ++++++++---------- util/reflect/struct_test.go | 4 ++-- 2 files changed, 10 insertions(+), 12 deletions(-) diff --git a/util/reflect/struct.go b/util/reflect/struct.go index 8342d21b..cfa136ff 100644 --- a/util/reflect/struct.go +++ b/util/reflect/struct.go @@ -46,9 +46,6 @@ func StructFieldByTag(src interface{}, tkey string, tval string) (interface{}, e if ts, ok := fld.Tag.Lookup(tkey); ok { for _, p := range strings.Split(ts, ",") { if p == tval { - if val.Kind() != reflect.Ptr && val.CanAddr() { - val = val.Addr() - } return val.Interface(), nil } } @@ -493,13 +490,14 @@ func btSplitter(str string) []string { } // queryToMap turns something like a[b][c]=4 into -// map[string]interface{}{ -// "a": map[string]interface{}{ -// "b": map[string]interface{}{ -// "c": 4, -// }, -// }, -// } +// +// map[string]interface{}{ +// "a": map[string]interface{}{ +// "b": map[string]interface{}{ +// "c": 4, +// }, +// }, +// } func queryToMap(param string) (map[string]interface{}, error) { rawKey, rawValue, err := splitKeyAndValue(param) if err != nil { diff --git a/util/reflect/struct_test.go b/util/reflect/struct_test.go index 00da531d..89be6de3 100644 --- a/util/reflect/struct_test.go +++ b/util/reflect/struct_test.go @@ -190,9 +190,9 @@ func TestStructFieldByTag(t *testing.T) { t.Fatal(err) } - if v, ok := iface.(*[]string); !ok { + if v, ok := iface.([]string); !ok { t.Fatalf("not *[]string %v", iface) - } else if len(*v) != 2 { + } else if len(v) != 2 { t.Fatalf("invalid number %v", iface) } } From 391813c2602a7e1be2c5d65bccab7ae9ef76d180 Mon Sep 17 00:00:00 2001 From: Vasiliy Tolstov Date: Sat, 4 May 2024 14:34:41 +0300 Subject: [PATCH 17/59] util/reflect: add StructFieldNameByTag Signed-off-by: Vasiliy Tolstov --- util/reflect/struct.go | 42 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 42 insertions(+) diff --git a/util/reflect/struct.go b/util/reflect/struct.go index cfa136ff..09132b08 100644 --- a/util/reflect/struct.go +++ b/util/reflect/struct.go @@ -25,6 +25,48 @@ type StructField struct { Field reflect.StructField } +// StructFieldNameByTag get struct field name by tag key and its value +func StructFieldNameByTag(src interface{}, tkey string, tval string) (string, error) { + sv := reflect.ValueOf(src) + if sv.Kind() == reflect.Ptr { + sv = sv.Elem() + } + if sv.Kind() != reflect.Struct { + return "", ErrInvalidStruct + } + + typ := sv.Type() + for idx := 0; idx < typ.NumField(); idx++ { + fld := typ.Field(idx) + val := sv.Field(idx) + if len(fld.PkgPath) != 0 { + continue + } + + if ts, ok := fld.Tag.Lookup(tkey); ok { + for _, p := range strings.Split(ts, ",") { + if p == tval { + return fld.Name, nil + } + } + } + + switch val.Kind() { + case reflect.Ptr: + if val = val.Elem(); val.Kind() == reflect.Struct { + if name, err := StructFieldNameByTag(val.Interface(), tkey, tval); err == nil { + return name, nil + } + } + case reflect.Struct: + if name, err := StructFieldNameByTag(val.Interface(), tkey, tval); err == nil { + return name, nil + } + } + } + return "", ErrNotFound +} + // StructFieldByTag get struct field by tag key and its value func StructFieldByTag(src interface{}, tkey string, tval string) (interface{}, error) { sv := reflect.ValueOf(src) From edc93e8c370fea9720bf50d6ed50bd2207522094 Mon Sep 17 00:00:00 2001 From: Vasiliy Tolstov Date: Sat, 4 May 2024 14:43:46 +0300 Subject: [PATCH 18/59] util/reflect: update StructFieldNameByTag Signed-off-by: Vasiliy Tolstov --- util/reflect/struct.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/util/reflect/struct.go b/util/reflect/struct.go index 09132b08..5fda8a84 100644 --- a/util/reflect/struct.go +++ b/util/reflect/struct.go @@ -26,13 +26,13 @@ type StructField struct { } // StructFieldNameByTag get struct field name by tag key and its value -func StructFieldNameByTag(src interface{}, tkey string, tval string) (string, error) { +func StructFieldNameByTag(src interface{}, tkey string, tval string) (string, interface{}, error) { sv := reflect.ValueOf(src) if sv.Kind() == reflect.Ptr { sv = sv.Elem() } if sv.Kind() != reflect.Struct { - return "", ErrInvalidStruct + return "", nil, ErrInvalidStruct } typ := sv.Type() @@ -46,7 +46,7 @@ func StructFieldNameByTag(src interface{}, tkey string, tval string) (string, er if ts, ok := fld.Tag.Lookup(tkey); ok { for _, p := range strings.Split(ts, ",") { if p == tval { - return fld.Name, nil + return fld.Name, val.Interface(), nil } } } @@ -54,17 +54,17 @@ func StructFieldNameByTag(src interface{}, tkey string, tval string) (string, er switch val.Kind() { case reflect.Ptr: if val = val.Elem(); val.Kind() == reflect.Struct { - if name, err := StructFieldNameByTag(val.Interface(), tkey, tval); err == nil { - return name, nil + if name, fld, err := StructFieldNameByTag(val.Interface(), tkey, tval); err == nil { + return name, fld, nil } } case reflect.Struct: - if name, err := StructFieldNameByTag(val.Interface(), tkey, tval); err == nil { - return name, nil + if name, fld, err := StructFieldNameByTag(val.Interface(), tkey, tval); err == nil { + return name, fld, nil } } } - return "", ErrNotFound + return "", nil, ErrNotFound } // StructFieldByTag get struct field by tag key and its value From 307a08f50cc3a40d3c94af757c26979e0b05356a Mon Sep 17 00:00:00 2001 From: Vasiliy Tolstov Date: Sat, 4 May 2024 15:31:08 +0300 Subject: [PATCH 19/59] add more checks Signed-off-by: Vasiliy Tolstov --- util/reflect/struct.go | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/util/reflect/struct.go b/util/reflect/struct.go index 5fda8a84..10266261 100644 --- a/util/reflect/struct.go +++ b/util/reflect/struct.go @@ -111,10 +111,21 @@ func StructFieldByTag(src interface{}, tkey string, tval string) (interface{}, e // ZeroFieldByPath clean struct field by its path func ZeroFieldByPath(src interface{}, path string) error { + if src == nil { + return nil + } var err error val := reflect.ValueOf(src) + if IsEmpty(val) { + return nil + } + for _, p := range strings.Split(path, ".") { + if IsEmpty(val) { + return nil + } + val, err = structValueByName(val, p) if err != nil { return err From cc0f24e01247234145aeede2736b52270cdb75b3 Mon Sep 17 00:00:00 2001 From: Vasiliy Tolstov Date: Sat, 4 May 2024 19:05:07 +0300 Subject: [PATCH 20/59] add ability to skip endpoints for tracer and meter Signed-off-by: Vasiliy Tolstov --- meter/meter.go | 7 +++++++ tracer/tracer.go | 12 +++++++++--- 2 files changed, 16 insertions(+), 3 deletions(-) diff --git a/meter/meter.go b/meter/meter.go index 887b8236..2afe65d5 100644 --- a/meter/meter.go +++ b/meter/meter.go @@ -24,6 +24,13 @@ var ( DefaultSummaryQuantiles = []float64{0.5, 0.9, 0.97, 0.99, 1} // DefaultSummaryWindow is the default window for summary DefaultSummaryWindow = 5 * time.Minute + // DefaultSkipEndpoints is the slice of endpoint that must not be metered + DefaultSkipEndpoints = []string{ + "MeterService.Metrics", + "HealthService.Live", + "HealthService.Ready", + "HealthService.Version", + } ) // Meter is an interface for collecting and instrumenting metrics diff --git a/tracer/tracer.go b/tracer/tracer.go index caf56960..8ad7cdf4 100644 --- a/tracer/tracer.go +++ b/tracer/tracer.go @@ -7,14 +7,20 @@ import ( "go.unistack.org/micro/v3/logger" ) -// DefaultTracer is the global default tracer -var DefaultTracer Tracer = NewTracer() - var ( + // DefaultTracer is the global default tracer + DefaultTracer Tracer = NewTracer() //nolint:revive // TraceIDKey is the key used for the trace id in the log call TraceIDKey = "trace-id" // SpanIDKey is the key used for the span id in the log call SpanIDKey = "span-id" + // DefaultSkipEndpoints is the slice of endpoint that must not be traced + DefaultSkipEndpoints = []string{ + "MeterService.Metrics", + "HealthService.Live", + "HealthService.Ready", + "HealthService.Version", + } ) func init() { From b08f5321b0439843ba1ba3df02d309b1ba710d89 Mon Sep 17 00:00:00 2001 From: Vasiliy Tolstov Date: Sat, 4 May 2024 19:18:12 +0300 Subject: [PATCH 21/59] tracer: allow to skip span recording Signed-off-by: Vasiliy Tolstov --- tracer/options.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/tracer/options.go b/tracer/options.go index a243f03e..4db47278 100644 --- a/tracer/options.go +++ b/tracer/options.go @@ -85,6 +85,7 @@ func (sk SpanKind) String() string { type SpanOptions struct { Labels []interface{} Kind SpanKind + Record bool } // SpanOption func signature @@ -116,6 +117,12 @@ func WithSpanKind(k SpanKind) SpanOption { } } +func WithSpanRecord(b bool) SpanOption { + return func(o *SpanOptions) { + o.Record = b + } +} + // Options struct type Options struct { // Context used to store custome tracer options From 7c0a5f5e2a2dd16229bc0c540a1caf4e9da60dc8 Mon Sep 17 00:00:00 2001 From: Vasiliy Tolstov Date: Sat, 4 May 2024 19:31:35 +0300 Subject: [PATCH 22/59] add abilit to skip span recording Signed-off-by: Vasiliy Tolstov --- tracer/options.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tracer/options.go b/tracer/options.go index 4db47278..6f736379 100644 --- a/tracer/options.go +++ b/tracer/options.go @@ -155,7 +155,8 @@ func NewEventOptions(opts ...EventOption) EventOptions { // NewSpanOptions returns default SpanOptions func NewSpanOptions(opts ...SpanOption) SpanOptions { options := SpanOptions{ - Kind: SpanKindInternal, + Kind: SpanKindInternal, + Record: true, } for _, o := range opts { o(&options) From 9c8fbb2202e6bb4174a85e715b2c204c33dae31d Mon Sep 17 00:00:00 2001 From: Vasiliy Tolstov Date: Sun, 5 May 2024 16:22:06 +0300 Subject: [PATCH 23/59] broker: add Event Context() method Signed-off-by: Vasiliy Tolstov --- broker/broker.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/broker/broker.go b/broker/broker.go index f716c947..28098188 100644 --- a/broker/broker.go +++ b/broker/broker.go @@ -88,6 +88,8 @@ type BatchHandler func(Events) error // Event is given to a subscription handler for processing type Event interface { + // Context return context.Context for event + Context() context.Context // Topic returns event topic Topic() string // Message returns broker message From 62074965eed794aafcba36934f054c83759fbf80 Mon Sep 17 00:00:00 2001 From: Vasiliy Tolstov Date: Thu, 9 May 2024 16:41:22 +0300 Subject: [PATCH 24/59] close #329 Signed-off-by: Vasiliy Tolstov --- logger/context.go | 11 +++++++++++ logger/options.go | 5 +++++ logger/slog/slog.go | 15 +++++++++++++++ meter/options.go | 12 ------------ semconv/logger.go | 4 ++++ 5 files changed, 35 insertions(+), 12 deletions(-) create mode 100644 semconv/logger.go diff --git a/logger/context.go b/logger/context.go index 6cb6b65f..848b9bca 100644 --- a/logger/context.go +++ b/logger/context.go @@ -4,6 +4,17 @@ import "context" type loggerKey struct{} +// MustContext returns logger from passed context or DefaultLogger if empty +func MustContext(ctx context.Context) Logger { + if ctx == nil { + return DefaultLogger + } + if l, ok := ctx.Value(loggerKey{}).(Logger); ok && l != nil { + return l + } + return DefaultLogger +} + // FromContext returns logger from passed context func FromContext(ctx context.Context) (Logger, bool) { if ctx == nil { diff --git a/logger/options.go b/logger/options.go index 3138892d..cd3b9f26 100644 --- a/logger/options.go +++ b/logger/options.go @@ -6,6 +6,8 @@ import ( "log/slog" "os" "time" + + "go.unistack.org/micro/v3/meter" ) // Option func signature @@ -45,6 +47,8 @@ type Options struct { Level Level // TimeFunc used to obtain current time TimeFunc func() time.Time + // Meter used to count logs for specific level + Meter meter.Meter } // NewOptions creates new options struct @@ -58,6 +62,7 @@ func NewOptions(opts ...Option) Options { ContextAttrFuncs: DefaultContextAttrFuncs, AddSource: true, TimeFunc: time.Now, + Meter: meter.DefaultMeter, } WithMicroKeys()(&options) diff --git a/logger/slog/slog.go b/logger/slog/slog.go index 3c699cff..76a7e1bb 100644 --- a/logger/slog/slog.go +++ b/logger/slog/slog.go @@ -11,6 +11,7 @@ import ( "sync" "go.unistack.org/micro/v3/logger" + "go.unistack.org/micro/v3/semconv" "go.unistack.org/micro/v3/tracer" ) @@ -150,6 +151,7 @@ func (s *slogLogger) Init(opts ...logger.Option) error { } func (s *slogLogger) Log(ctx context.Context, lvl logger.Level, attrs ...interface{}) { + s.opts.Meter.Counter(semconv.LoggerMessageTotal, "level", lvl.String()).Inc() if !s.V(lvl) { return } @@ -189,6 +191,7 @@ func (s *slogLogger) Log(ctx context.Context, lvl logger.Level, attrs ...interfa } func (s *slogLogger) Logf(ctx context.Context, lvl logger.Level, msg string, attrs ...interface{}) { + s.opts.Meter.Counter(semconv.LoggerMessageTotal, "level", lvl.String()).Inc() if !s.V(lvl) { return } @@ -228,6 +231,7 @@ func (s *slogLogger) Logf(ctx context.Context, lvl logger.Level, msg string, att } func (s *slogLogger) Info(ctx context.Context, attrs ...interface{}) { + s.opts.Meter.Counter(semconv.LoggerMessageTotal, "level", logger.InfoLevel.String()).Inc() if !s.V(logger.InfoLevel) { return } @@ -249,6 +253,7 @@ func (s *slogLogger) Info(ctx context.Context, attrs ...interface{}) { } func (s *slogLogger) Infof(ctx context.Context, msg string, attrs ...interface{}) { + s.opts.Meter.Counter(semconv.LoggerMessageTotal, "level", logger.InfoLevel.String()).Inc() if !s.V(logger.InfoLevel) { return } @@ -270,6 +275,7 @@ func (s *slogLogger) Infof(ctx context.Context, msg string, attrs ...interface{} } func (s *slogLogger) Debug(ctx context.Context, attrs ...interface{}) { + s.opts.Meter.Counter(semconv.LoggerMessageTotal, "level", logger.DebugLevel.String()).Inc() if !s.V(logger.DebugLevel) { return } @@ -291,6 +297,7 @@ func (s *slogLogger) Debug(ctx context.Context, attrs ...interface{}) { } func (s *slogLogger) Debugf(ctx context.Context, msg string, attrs ...interface{}) { + s.opts.Meter.Counter(semconv.LoggerMessageTotal, "level", logger.DebugLevel.String()).Inc() if !s.V(logger.DebugLevel) { return } @@ -312,6 +319,7 @@ func (s *slogLogger) Debugf(ctx context.Context, msg string, attrs ...interface{ } func (s *slogLogger) Trace(ctx context.Context, attrs ...interface{}) { + s.opts.Meter.Counter(semconv.LoggerMessageTotal, "level", logger.TraceLevel.String()).Inc() if !s.V(logger.TraceLevel) { return } @@ -333,6 +341,7 @@ func (s *slogLogger) Trace(ctx context.Context, attrs ...interface{}) { } func (s *slogLogger) Tracef(ctx context.Context, msg string, attrs ...interface{}) { + s.opts.Meter.Counter(semconv.LoggerMessageTotal, "level", logger.TraceLevel.String()).Inc() if !s.V(logger.TraceLevel) { return } @@ -354,6 +363,7 @@ func (s *slogLogger) Tracef(ctx context.Context, msg string, attrs ...interface{ } func (s *slogLogger) Error(ctx context.Context, attrs ...interface{}) { + s.opts.Meter.Counter(semconv.LoggerMessageTotal, "level", logger.ErrorLevel.String()).Inc() if !s.V(logger.ErrorLevel) { return } @@ -393,6 +403,7 @@ func (s *slogLogger) Error(ctx context.Context, attrs ...interface{}) { } func (s *slogLogger) Errorf(ctx context.Context, msg string, attrs ...interface{}) { + s.opts.Meter.Counter(semconv.LoggerMessageTotal, "level", logger.ErrorLevel.String()).Inc() if !s.V(logger.ErrorLevel) { return } @@ -432,6 +443,7 @@ func (s *slogLogger) Errorf(ctx context.Context, msg string, attrs ...interface{ } func (s *slogLogger) Fatal(ctx context.Context, attrs ...interface{}) { + s.opts.Meter.Counter(semconv.LoggerMessageTotal, "level", logger.FatalLevel.String()).Inc() if !s.V(logger.FatalLevel) { return } @@ -454,6 +466,7 @@ func (s *slogLogger) Fatal(ctx context.Context, attrs ...interface{}) { } func (s *slogLogger) Fatalf(ctx context.Context, msg string, attrs ...interface{}) { + s.opts.Meter.Counter(semconv.LoggerMessageTotal, "level", logger.FatalLevel.String()).Inc() if !s.V(logger.FatalLevel) { return } @@ -476,6 +489,7 @@ func (s *slogLogger) Fatalf(ctx context.Context, msg string, attrs ...interface{ } func (s *slogLogger) Warn(ctx context.Context, attrs ...interface{}) { + s.opts.Meter.Counter(semconv.LoggerMessageTotal, "level", logger.WarnLevel.String()).Inc() if !s.V(logger.WarnLevel) { return } @@ -497,6 +511,7 @@ func (s *slogLogger) Warn(ctx context.Context, attrs ...interface{}) { } func (s *slogLogger) Warnf(ctx context.Context, msg string, attrs ...interface{}) { + s.opts.Meter.Counter(semconv.LoggerMessageTotal, "level", logger.WarnLevel.String()).Inc() if !s.V(logger.WarnLevel) { return } diff --git a/meter/options.go b/meter/options.go index 14bcfc69..79ef2a11 100644 --- a/meter/options.go +++ b/meter/options.go @@ -2,8 +2,6 @@ package meter import ( "context" - - "go.unistack.org/micro/v3/logger" ) // Option powers the configuration for metrics implementations: @@ -11,8 +9,6 @@ type Option func(*Options) // Options for metrics implementations type Options struct { - // Logger used for logging - Logger logger.Logger // Context holds external options Context context.Context // Name holds the meter name @@ -39,7 +35,6 @@ func NewOptions(opt ...Option) Options { Address: DefaultAddress, Path: DefaultPath, Context: context.Background(), - Logger: logger.DefaultLogger, MetricPrefix: DefaultMetricPrefix, LabelPrefix: DefaultLabelPrefix, } @@ -95,13 +90,6 @@ func TimingObjectives(value map[float64]float64) Option { } */ -// Logger sets the logger -func Logger(l logger.Logger) Option { - return func(o *Options) { - o.Logger = l - } -} - // Labels sets the meter labels func Labels(ls ...string) Option { return func(o *Options) { diff --git a/semconv/logger.go b/semconv/logger.go new file mode 100644 index 00000000..88ef668f --- /dev/null +++ b/semconv/logger.go @@ -0,0 +1,4 @@ +package semconv + +// LoggerMessageTotal specifies meter metric name for logger messages +var LoggerMessageTotal = "logger_message_total" From fa1427014c28f19614f37f973c76ae4798858c63 Mon Sep 17 00:00:00 2001 From: Vasiliy Tolstov Date: Thu, 9 May 2024 19:16:12 +0300 Subject: [PATCH 25/59] close #343 Signed-off-by: Vasiliy Tolstov --- tracer/options.go | 15 ++++++++++++--- tracer/tracer.go | 2 ++ 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/tracer/options.go b/tracer/options.go index 6f736379..98c4d3a7 100644 --- a/tracer/options.go +++ b/tracer/options.go @@ -83,9 +83,11 @@ func (sk SpanKind) String() string { // SpanOptions contains span option type SpanOptions struct { - Labels []interface{} - Kind SpanKind - Record bool + StatusMsg string + Labels []interface{} + Status SpanStatus + Kind SpanKind + Record bool } // SpanOption func signature @@ -111,6 +113,13 @@ func WithSpanLabels(kv ...interface{}) SpanOption { } } +func WithSpanStatus(st SpanStatus, msg string) SpanOption { + return func(o *SpanOptions) { + o.Status = st + o.StatusMsg = msg + } +} + func WithSpanKind(k SpanKind) SpanOption { return func(o *SpanOptions) { o.Kind = k diff --git a/tracer/tracer.go b/tracer/tracer.go index 8ad7cdf4..a5100dca 100644 --- a/tracer/tracer.go +++ b/tracer/tracer.go @@ -44,6 +44,8 @@ type Tracer interface { Init(...Option) error // Start a trace Start(ctx context.Context, name string, opts ...SpanOption) (context.Context, Span) + // Extract get span metadata from context + // Extract(ctx context.Context) // Flush flushes spans Flush(ctx context.Context) error } From 3eebfb5b1197e1d44f2a75a6a18efa0e249161fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=D0=92=D0=B0=D1=81=D0=B8=D0=BB=D0=B8=D0=B9=20=D0=A2=D0=BE?= =?UTF-8?q?=D0=BB=D1=81=D1=82=D0=BE=D0=B2?= Date: Fri, 10 May 2024 08:12:10 +0300 Subject: [PATCH 26/59] =?UTF-8?q?=D0=9E=D0=B1=D0=BD=D0=BE=D0=B2=D0=B8?= =?UTF-8?q?=D1=82=D1=8C=20options.go?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit --- options.go | 10 +--------- 1 file changed, 1 insertion(+), 9 deletions(-) diff --git a/options.go b/options.go index 37164a71..97adbf5e 100644 --- a/options.go +++ b/options.go @@ -269,15 +269,7 @@ func Logger(l logger.Logger, opts ...LoggerOption) Option { } } } - for _, mtr := range o.Meters { - for _, or := range lopts.meters { - if mtr.Name() == or || all { - if err = mtr.Init(meter.Logger(l)); err != nil { - return err - } - } - } - } + for _, trc := range o.Tracers { for _, ot := range lopts.tracers { if trc.Name() == ot || all { From c65afcea1b99dfd0864367ca399192d1b1e8e70e Mon Sep 17 00:00:00 2001 From: devstigneev Date: Mon, 1 Jul 2024 09:47:51 +0300 Subject: [PATCH 27/59] fix impl interface --- broker/memory/memory.go | 4 ++++ network/tunnel/broker/broker.go | 4 ++++ 2 files changed, 8 insertions(+) diff --git a/broker/memory/memory.go b/broker/memory/memory.go index 1b301baf..9fa1551c 100644 --- a/broker/memory/memory.go +++ b/broker/memory/memory.go @@ -373,6 +373,10 @@ func (m *memoryEvent) SetError(err error) { m.err = err } +func (m *memoryEvent) Context() context.Context { + return m.opts.Context +} + func (m *memorySubscriber) Options() broker.SubscribeOptions { return m.opts } diff --git a/network/tunnel/broker/broker.go b/network/tunnel/broker/broker.go index ea14c1f9..3066bd44 100644 --- a/network/tunnel/broker/broker.go +++ b/network/tunnel/broker/broker.go @@ -305,6 +305,10 @@ func (t *tunEvent) SetError(err error) { t.err = err } +func (t *tunEvent) Context() context.Context { + return context.TODO() +} + // NewBroker returns new tunnel broker func NewBroker(opts ...broker.Option) (broker.Broker, error) { options := broker.NewOptions(opts...) From 00b3ceb468cb6f7ad95ec8939f38cc18affac8e3 Mon Sep 17 00:00:00 2001 From: Vasiliy Tolstov Date: Thu, 4 Jul 2024 14:56:48 +0300 Subject: [PATCH 28/59] smeconv: fix naming Signed-off-by: Vasiliy Tolstov --- semconv/cache.go | 12 ------------ semconv/store.go | 12 ++++++++++++ 2 files changed, 12 insertions(+), 12 deletions(-) delete mode 100644 semconv/cache.go create mode 100644 semconv/store.go diff --git a/semconv/cache.go b/semconv/cache.go deleted file mode 100644 index f12739c1..00000000 --- a/semconv/cache.go +++ /dev/null @@ -1,12 +0,0 @@ -package semconv - -var ( - // CacheRequestDurationSeconds specifies meter metric name - CacheRequestDurationSeconds = "cache_request_duration_seconds" - // ClientRequestLatencyMicroseconds specifies meter metric name - CacheRequestLatencyMicroseconds = "cache_request_latency_microseconds" - // CacheRequestTotal specifies meter metric name - CacheRequestTotal = "cache_request_total" - // CacheRequestInflight specifies meter metric name - CacheRequestInflight = "cache_request_inflight" -) diff --git a/semconv/store.go b/semconv/store.go new file mode 100644 index 00000000..f3828a85 --- /dev/null +++ b/semconv/store.go @@ -0,0 +1,12 @@ +package semconv + +var ( + // StoreRequestDurationSeconds specifies meter metric name + StoreRequestDurationSeconds = "store_request_duration_seconds" + // ClientRequestLatencyMicroseconds specifies meter metric name + StoreRequestLatencyMicroseconds = "store_request_latency_microseconds" + // StoreRequestTotal specifies meter metric name + StoreRequestTotal = "store_request_total" + // StoreRequestInflight specifies meter metric name + StoreRequestInflight = "store_request_inflight" +) From 35e62fbeb0d23bb311dce59f6965d9ddac5d6a8b Mon Sep 17 00:00:00 2001 From: Vasiliy Tolstov Date: Sat, 6 Jul 2024 00:09:27 +0300 Subject: [PATCH 29/59] tracer: add default context attr funcs option Signed-off-by: Vasiliy Tolstov --- logger/logger.go | 2 +- logger/options.go | 2 +- tracer/options.go | 7 +++++-- tracer/tracer.go | 3 +++ 4 files changed, 10 insertions(+), 4 deletions(-) diff --git a/logger/logger.go b/logger/logger.go index 71dcc2d2..175311bc 100644 --- a/logger/logger.go +++ b/logger/logger.go @@ -1,5 +1,5 @@ // Package logger provides a log interface -package logger // import "go.unistack.org/micro/v3/logger" +package logger import ( "context" diff --git a/logger/options.go b/logger/options.go index cd3b9f26..a7008868 100644 --- a/logger/options.go +++ b/logger/options.go @@ -74,7 +74,7 @@ func NewOptions(opts ...Option) Options { return options } -// WithContextAttrFuncs appends default funcs for the context arrts filler +// WithContextAttrFuncs appends default funcs for the context attrs filler func WithContextAttrFuncs(fncs ...ContextAttrFunc) Option { return func(o *Options) { o.ContextAttrFuncs = append(o.ContextAttrFuncs, fncs...) diff --git a/tracer/options.go b/tracer/options.go index 98c4d3a7..1c67e1e8 100644 --- a/tracer/options.go +++ b/tracer/options.go @@ -140,6 +140,8 @@ type Options struct { Logger logger.Logger // Name of the tracer Name string + // ContextAttrFuncs contains funcs that provides tracing + ContextAttrFuncs []ContextAttrFunc } // Option func signature @@ -176,8 +178,9 @@ func NewSpanOptions(opts ...SpanOption) SpanOptions { // NewOptions returns default options func NewOptions(opts ...Option) Options { options := Options{ - Logger: logger.DefaultLogger, - Context: context.Background(), + Logger: logger.DefaultLogger, + Context: context.Background(), + ContextAttrFuncs: DefaultContextAttrFuncs, } for _, o := range opts { o(&options) diff --git a/tracer/tracer.go b/tracer/tracer.go index a5100dca..8951da8a 100644 --- a/tracer/tracer.go +++ b/tracer/tracer.go @@ -21,8 +21,11 @@ var ( "HealthService.Ready", "HealthService.Version", } + DefaultContextAttrFuncs []ContextAttrFunc ) +type ContextAttrFunc func(ctx context.Context) []interface{} + func init() { logger.DefaultContextAttrFuncs = append(logger.DefaultContextAttrFuncs, func(ctx context.Context) []interface{} { From 6c68d390815732a0ed437cedaf4d19496d7378c8 Mon Sep 17 00:00:00 2001 From: Vasiliy Tolstov Date: Thu, 1 Aug 2024 01:06:02 +0300 Subject: [PATCH 30/59] errors: add RFC9457 problem type closes #297 Signed-off-by: Vasiliy Tolstov --- errors/errors.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/errors/errors.go b/errors/errors.go index b59034e6..3224515d 100644 --- a/errors/errors.go +++ b/errors/errors.go @@ -44,6 +44,20 @@ var ( ErrGatewayTimeout = &Error{Code: 504} ) +const ProblemContentType = "application/problem+json" + +type Problem struct { + Type string `json:"type,omitempty"` + Title string `json:"title,omitempty"` + Detail string `json:"detail,omitempty"` + Instance string `json:"instance,omitempty"` + Errors []struct { + Title string `json:"title,omitempty"` + Detail string `json:"detail,omitempty"` + } `json:"errors,omitempty"` + Status int `json:"status,omitempty"` +} + // Error type type Error struct { // ID holds error id or service, usually someting like my_service or id From 1a9236caad75b106e8525b9c0d964dc3c98fb578 Mon Sep 17 00:00:00 2001 From: Vasiliy Tolstov Date: Wed, 4 Sep 2024 22:41:05 +0300 Subject: [PATCH 31/59] update meter options Signed-off-by: Vasiliy Tolstov --- logger/options.go | 7 +++++++ util/xpool/pool.go | 13 ++++++++++++- 2 files changed, 19 insertions(+), 1 deletion(-) diff --git a/logger/options.go b/logger/options.go index a7008868..e1113daf 100644 --- a/logger/options.go +++ b/logger/options.go @@ -137,6 +137,13 @@ func WithName(n string) Option { } } +// WithMeter sets the meter +func WithMeter(m meter.Meter) Option { + return func(o *Options) { + o.Meter = m + } +} + // WithTimeFunc sets the func to obtain current time func WithTimeFunc(fn func() time.Time) Option { return func(o *Options) { diff --git a/util/xpool/pool.go b/util/xpool/pool.go index 1ffe4293..cfc8d5f0 100644 --- a/util/xpool/pool.go +++ b/util/xpool/pool.go @@ -1,6 +1,9 @@ package pool -import "sync" +import ( + "bytes" + "sync" +) type Pool[T any] struct { p *sync.Pool @@ -23,3 +26,11 @@ func (p Pool[T]) Get() T { func (p Pool[T]) Put(t T) { p.p.Put(t) } + +func NewBytePool(size int) Pool[T] { + return NewPool(func() []byte { return make([]byte, size) }) +} + +func NewBytesPool() Pool[T] { + return NewPool(func() *bytes.Buffer { return bytes.NewBuffer(nil) }) +} From 53b40617e222fbb71cc849ee6a36b26d8ce8ed74 Mon Sep 17 00:00:00 2001 From: Vasiliy Tolstov Date: Wed, 4 Sep 2024 23:06:40 +0300 Subject: [PATCH 32/59] fixup util/xpool Signed-off-by: Vasiliy Tolstov --- util/xpool/pool.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/util/xpool/pool.go b/util/xpool/pool.go index cfc8d5f0..fa27df3a 100644 --- a/util/xpool/pool.go +++ b/util/xpool/pool.go @@ -27,10 +27,10 @@ func (p Pool[T]) Put(t T) { p.p.Put(t) } -func NewBytePool(size int) Pool[T] { +func NewBytePool(size int) Pool[[]byte] { return NewPool(func() []byte { return make([]byte, size) }) } -func NewBytesPool() Pool[T] { +func NewBytesPool() Pool[*bytes.Buffer] { return NewPool(func() *bytes.Buffer { return bytes.NewBuffer(nil) }) } From 3ace7657dc4541da668936b1b8d06c3330f84365 Mon Sep 17 00:00:00 2001 From: Vasiliy Tolstov Date: Tue, 10 Sep 2024 10:43:45 +0300 Subject: [PATCH 33/59] codec: RawMessage Marshal fix Signed-off-by: Vasiliy Tolstov --- codec/codec.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/codec/codec.go b/codec/codec.go index 20231315..daaf6ee1 100644 --- a/codec/codec.go +++ b/codec/codec.go @@ -93,6 +93,8 @@ type RawMessage []byte func (m *RawMessage) MarshalJSON() ([]byte, error) { if m == nil { return []byte("null"), nil + } else if len(*m) == 0 { + return []byte("null"), nil } return *m, nil } From d9b2f2a45dc400f1c0011b969cd6e85479d39a72 Mon Sep 17 00:00:00 2001 From: Gorbunov Kirill Andreevich Date: Mon, 16 Sep 2024 14:48:47 +0300 Subject: [PATCH 34/59] #347 add test --- util/sort/sort_test.go | 60 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 60 insertions(+) create mode 100644 util/sort/sort_test.go diff --git a/util/sort/sort_test.go b/util/sort/sort_test.go new file mode 100644 index 00000000..2c85bcd1 --- /dev/null +++ b/util/sort/sort_test.go @@ -0,0 +1,60 @@ +package sort + +import ( + "reflect" + "testing" +) + +func TestUniq(t *testing.T) { + type args struct { + labels []interface{} + } + + tests := []struct { + name string + args args + want []interface{} + }{ + { + name: "test#1", + args: args{ + labels: append(make([]interface{}, 0), "test-1", 1, "test-2", 2), + }, + want: append(make([]interface{}, 0), "test-1", 1, "test-2", 2), + }, + { + name: "test#2", + args: args{ + labels: append(make([]interface{}, 0), "test-1", 1, "test-2", 2, "test-2", 2), + }, + want: append(make([]interface{}, 0), "test-1", 1, "test-2", 2), + }, + { + name: "test#3", + args: args{ + labels: append(make([]interface{}, 0), "test-1", 1, "test-2", 2, "test-2", 3), + }, + want: append(make([]interface{}, 0), "test-1", 1, "test-2", 3), + }, + { + name: "test#4", + args: args{ + labels: append(make([]interface{}, 0), + "test-1", 1, "test-1", 2, + "test-2", 3, "test-2", 2, + "test-3", 5, "test-3", 3, + "test-1", 4, "test-1", 1), + }, + want: append(make([]interface{}, 0), "test-1", 1, "test-2", 2, "test-3", 3), + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var got []interface{} + if got = Uniq(tt.args.labels); !reflect.DeepEqual(got, tt.want) { + t.Errorf("Uniq() = %v, want %v", got, tt.want) + } + t.Logf("got-%#v", got) + }) + } +} From eb1482d78929fabc5587aa3edaf1352606659853 Mon Sep 17 00:00:00 2001 From: Vasiliy Tolstov Date: Mon, 16 Sep 2024 22:41:36 +0300 Subject: [PATCH 35/59] codec: simplify codec interface Signed-off-by: Vasiliy Tolstov --- codec/codec.go | 42 +------------------------------ codec/noop.go | 68 +++++--------------------------------------------- server/noop.go | 3 +-- 3 files changed, 8 insertions(+), 105 deletions(-) diff --git a/codec/codec.go b/codec/codec.go index daaf6ee1..3de1899a 100644 --- a/codec/codec.go +++ b/codec/codec.go @@ -3,17 +3,6 @@ package codec // import "go.unistack.org/micro/v3/codec" import ( "errors" - "io" - - "go.unistack.org/micro/v3/metadata" -) - -// Message types -const ( - Error MessageType = iota - Request - Response - Event ) var ( @@ -32,42 +21,13 @@ var ( DefaultTagName = "codec" ) -// MessageType specifies message type for codec -type MessageType int - -// Codec encodes/decodes various types of messages used within micro. -// ReadHeader and ReadBody are called in pairs to read requests/responses -// from the connection. Close is called when finished with the -// connection. ReadBody may be called with a nil argument to force the -// body to be read and discarded. +// Codec encodes/decodes various types of messages. type Codec interface { - ReadHeader(r io.Reader, m *Message, mt MessageType) error - ReadBody(r io.Reader, v interface{}) error - Write(w io.Writer, m *Message, v interface{}) error Marshal(v interface{}, opts ...Option) ([]byte, error) Unmarshal(b []byte, v interface{}, opts ...Option) error String() string } -// Message represents detailed information about -// the communication, likely followed by the body. -// In the case of an error, body may be nil. -type Message struct { - Header metadata.Metadata - Target string - Method string - Endpoint string - Error string - ID string - Body []byte - Type MessageType -} - -// NewMessage creates new codec message -func NewMessage(t MessageType) *Message { - return &Message{Type: t, Header: metadata.New(0)} -} - // MarshalAppend calls codec.Marshal(v) and returns the data appended to buf. // If codec implements MarshalAppend, that is called instead. func MarshalAppend(buf []byte, c Codec, v interface{}, opts ...Option) ([]byte, error) { diff --git a/codec/noop.go b/codec/noop.go index c9a66e9b..76d21d93 100644 --- a/codec/noop.go +++ b/codec/noop.go @@ -2,70 +2,14 @@ package codec import ( "encoding/json" - "io" + + codecpb "go.unistack.org/micro-proto/v3/codec" ) type noopCodec struct { opts Options } -func (c *noopCodec) ReadHeader(conn io.Reader, m *Message, t MessageType) error { - return nil -} - -func (c *noopCodec) ReadBody(conn io.Reader, b interface{}) error { - // read bytes - buf, err := io.ReadAll(conn) - if err != nil { - return err - } - - if b == nil { - return nil - } - - switch v := b.(type) { - case *string: - *v = string(buf) - case *[]byte: - *v = buf - case *Frame: - v.Data = buf - default: - return json.Unmarshal(buf, v) - } - - return nil -} - -func (c *noopCodec) Write(conn io.Writer, m *Message, b interface{}) error { - if b == nil { - return nil - } - - var v []byte - switch vb := b.(type) { - case *Frame: - v = vb.Data - case string: - v = []byte(vb) - case *string: - v = []byte(*vb) - case *[]byte: - v = *vb - case []byte: - v = vb - default: - var err error - v, err = json.Marshal(vb) - if err != nil { - return err - } - } - _, err := conn.Write(v) - return err -} - func (c *noopCodec) String() string { return "noop" } @@ -91,8 +35,8 @@ func (c *noopCodec) Marshal(v interface{}, opts ...Option) ([]byte, error) { return ve, nil case *Frame: return ve.Data, nil - case *Message: - return ve.Body, nil + case *codecpb.Frame: + return ve.Data, nil } return json.Marshal(v) @@ -115,8 +59,8 @@ func (c *noopCodec) Unmarshal(d []byte, v interface{}, opts ...Option) error { case *Frame: ve.Data = d return nil - case *Message: - ve.Body = d + case *codecpb.Frame: + ve.Data = d return nil } diff --git a/server/noop.go b/server/noop.go index 956ac9c2..6d2ecc05 100644 --- a/server/noop.go +++ b/server/noop.go @@ -1,7 +1,6 @@ package server import ( - "bytes" "context" "fmt" "reflect" @@ -691,7 +690,7 @@ func (n *noopServer) createSubHandler(sb *subscriber, opts Options) broker.Handl req = req.Elem() } - if err = cf.ReadBody(bytes.NewBuffer(msg.Body), req.Interface()); err != nil { + if err = cf.Unmarshal(msg.Body, req.Interface()); err != nil { return err } From 3aaf1182cb58524ef5193c816f3bfafb31bdf99b Mon Sep 17 00:00:00 2001 From: Vasiliy Tolstov Date: Mon, 16 Sep 2024 23:02:45 +0300 Subject: [PATCH 36/59] add codec option Signed-off-by: Vasiliy Tolstov --- codec/codec.go | 23 +++++------------------ codec/options.go | 23 ++++++++--------------- 2 files changed, 13 insertions(+), 33 deletions(-) diff --git a/codec/codec.go b/codec/codec.go index 3de1899a..b850cf8f 100644 --- a/codec/codec.go +++ b/codec/codec.go @@ -1,5 +1,5 @@ // Package codec is an interface for encoding messages -package codec // import "go.unistack.org/micro/v3/codec" +package codec import ( "errors" @@ -13,8 +13,6 @@ var ( ) var ( - // DefaultMaxMsgSize specifies how much data codec can handle - DefaultMaxMsgSize = 1024 * 1024 * 4 // 4Mb // DefaultCodec is the global default codec DefaultCodec = NewCodec() // DefaultTagName specifies struct tag name to control codec Marshal/Unmarshal @@ -28,21 +26,10 @@ type Codec interface { String() string } -// MarshalAppend calls codec.Marshal(v) and returns the data appended to buf. -// If codec implements MarshalAppend, that is called instead. -func MarshalAppend(buf []byte, c Codec, v interface{}, opts ...Option) ([]byte, error) { - if nc, ok := c.(interface { - MarshalAppend([]byte, interface{}, ...Option) ([]byte, error) - }); ok { - return nc.MarshalAppend(buf, v, opts...) - } - - mbuf, err := c.Marshal(v, opts...) - if err != nil { - return nil, err - } - - return append(buf, mbuf...), nil +type CodecV2 interface { + Marshal(buf []byte, v interface{}, opts ...Option) ([]byte, error) + Unmarshal(buf []byte, v interface{}, opts ...Option) error + String() string } // RawMessage is a raw encoded JSON value. diff --git a/codec/options.go b/codec/options.go index afcfb1c5..a0ed6e54 100644 --- a/codec/options.go +++ b/codec/options.go @@ -23,15 +23,8 @@ type Options struct { Context context.Context // TagName specifies tag name in struct to control codec TagName string - // MaxMsgSize specifies max messages size that reads by codec - MaxMsgSize int -} - -// MaxMsgSize sets the max message size -func MaxMsgSize(n int) Option { - return func(o *Options) { - o.MaxMsgSize = n - } + // Flatten specifies that struct must be analyzed for flatten tag + Flatten bool } // TagName sets the codec tag name in struct @@ -65,12 +58,12 @@ func Meter(m meter.Meter) Option { // NewOptions returns new options func NewOptions(opts ...Option) Options { options := Options{ - Context: context.Background(), - Logger: logger.DefaultLogger, - Meter: meter.DefaultMeter, - Tracer: tracer.DefaultTracer, - MaxMsgSize: DefaultMaxMsgSize, - TagName: DefaultTagName, + Context: context.Background(), + Logger: logger.DefaultLogger, + Meter: meter.DefaultMeter, + Tracer: tracer.DefaultTracer, + TagName: DefaultTagName, + Flatten: false, } for _, o := range opts { From a98618ed5b8ea4e6ddbac3979fcf51db9638aa9f Mon Sep 17 00:00:00 2001 From: Vasiliy Tolstov Date: Mon, 16 Sep 2024 23:10:43 +0300 Subject: [PATCH 37/59] add codec.Flatten option Signed-off-by: Vasiliy Tolstov --- codec/options.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/codec/options.go b/codec/options.go index a0ed6e54..3906a82c 100644 --- a/codec/options.go +++ b/codec/options.go @@ -34,6 +34,13 @@ func TagName(n string) Option { } } +// Flatten enables checking for flatten tag name +func Flatten(b bool) Option { + return func(o *Options) { + o.Flatten = b + } +} + // Logger sets the logger func Logger(l logger.Logger) Option { return func(o *Options) { From 4ae72771403e8bbbc195df20b500934eca229532 Mon Sep 17 00:00:00 2001 From: Vasiliy Tolstov Date: Fri, 20 Sep 2024 08:27:25 +0300 Subject: [PATCH 38/59] meter: remove prefix options Signed-off-by: Vasiliy Tolstov --- meter/options.go | 28 ++++------------------------ 1 file changed, 4 insertions(+), 24 deletions(-) diff --git a/meter/options.go b/meter/options.go index 79ef2a11..236bebd3 100644 --- a/meter/options.go +++ b/meter/options.go @@ -17,10 +17,6 @@ type Options struct { Address string // Path holds the path for metrics Path string - // MetricPrefix holds the prefix for all metrics - MetricPrefix string - // LabelPrefix holds the prefix for all labels - LabelPrefix string // Labels holds the default labels Labels []string // WriteProcessMetrics flag to write process metrics @@ -32,11 +28,9 @@ type Options struct { // NewOptions prepares a set of options: func NewOptions(opt ...Option) Options { opts := Options{ - Address: DefaultAddress, - Path: DefaultPath, - Context: context.Background(), - MetricPrefix: DefaultMetricPrefix, - LabelPrefix: DefaultLabelPrefix, + Address: DefaultAddress, + Path: DefaultPath, + Context: context.Background(), } for _, o := range opt { @@ -46,20 +40,6 @@ func NewOptions(opt ...Option) Options { return opts } -// LabelPrefix sets the labels prefix -func LabelPrefix(pref string) Option { - return func(o *Options) { - o.LabelPrefix = pref - } -} - -// MetricPrefix sets the metric prefix -func MetricPrefix(pref string) Option { - return func(o *Options) { - o.MetricPrefix = pref - } -} - // Context sets the metrics context func Context(ctx context.Context) Option { return func(o *Options) { @@ -90,7 +70,7 @@ func TimingObjectives(value map[float64]float64) Option { } */ -// Labels sets the meter labels +// Labels add the meter labels func Labels(ls ...string) Option { return func(o *Options) { o.Labels = append(o.Labels, ls...) From de9e4d73f556acdc2d676272b51dd1e6a6885bae Mon Sep 17 00:00:00 2001 From: Vasiliy Tolstov Date: Fri, 20 Sep 2024 08:38:36 +0300 Subject: [PATCH 39/59] change semconv metric names to include micro_ prefix Signed-off-by: Vasiliy Tolstov --- semconv/broker.go | 18 +++++++++--------- semconv/client.go | 8 ++++---- semconv/logger.go | 2 +- semconv/server.go | 8 ++++---- semconv/store.go | 8 ++++---- 5 files changed, 22 insertions(+), 22 deletions(-) diff --git a/semconv/broker.go b/semconv/broker.go index 4c949225..f9f0d751 100644 --- a/semconv/broker.go +++ b/semconv/broker.go @@ -2,21 +2,21 @@ package semconv var ( // PublishMessageDurationSeconds specifies meter metric name - PublishMessageDurationSeconds = "publish_message_duration_seconds" + PublishMessageDurationSeconds = "micro_publish_message_duration_seconds" // PublishMessageLatencyMicroseconds specifies meter metric name - PublishMessageLatencyMicroseconds = "publish_message_latency_microseconds" + PublishMessageLatencyMicroseconds = "micro_publish_message_latency_microseconds" // PublishMessageTotal specifies meter metric name - PublishMessageTotal = "publish_message_total" + PublishMessageTotal = "micro_publish_message_total" // PublishMessageInflight specifies meter metric name - PublishMessageInflight = "publish_message_inflight" + PublishMessageInflight = "micro_publish_message_inflight" // SubscribeMessageDurationSeconds specifies meter metric name - SubscribeMessageDurationSeconds = "subscribe_message_duration_seconds" + SubscribeMessageDurationSeconds = "micro_subscribe_message_duration_seconds" // SubscribeMessageLatencyMicroseconds specifies meter metric name - SubscribeMessageLatencyMicroseconds = "subscribe_message_latency_microseconds" + SubscribeMessageLatencyMicroseconds = "micro_subscribe_message_latency_microseconds" // SubscribeMessageTotal specifies meter metric name - SubscribeMessageTotal = "subscribe_message_total" + SubscribeMessageTotal = "micro_subscribe_message_total" // SubscribeMessageInflight specifies meter metric name - SubscribeMessageInflight = "subscribe_message_inflight" + SubscribeMessageInflight = "micro_subscribe_message_inflight" // BrokerGroupLag specifies broker lag - BrokerGroupLag = "broker_group_lag" + BrokerGroupLag = "micro_broker_group_lag" ) diff --git a/semconv/client.go b/semconv/client.go index 96671471..006854db 100644 --- a/semconv/client.go +++ b/semconv/client.go @@ -2,11 +2,11 @@ package semconv var ( // ClientRequestDurationSeconds specifies meter metric name - ClientRequestDurationSeconds = "client_request_duration_seconds" + ClientRequestDurationSeconds = "micro_client_request_duration_seconds" // ClientRequestLatencyMicroseconds specifies meter metric name - ClientRequestLatencyMicroseconds = "client_request_latency_microseconds" + ClientRequestLatencyMicroseconds = "micro_client_request_latency_microseconds" // ClientRequestTotal specifies meter metric name - ClientRequestTotal = "client_request_total" + ClientRequestTotal = "micro_client_request_total" // ClientRequestInflight specifies meter metric name - ClientRequestInflight = "client_request_inflight" + ClientRequestInflight = "micro_client_request_inflight" ) diff --git a/semconv/logger.go b/semconv/logger.go index 88ef668f..3a20f0b4 100644 --- a/semconv/logger.go +++ b/semconv/logger.go @@ -1,4 +1,4 @@ package semconv // LoggerMessageTotal specifies meter metric name for logger messages -var LoggerMessageTotal = "logger_message_total" +var LoggerMessageTotal = "micro_logger_message_total" diff --git a/semconv/server.go b/semconv/server.go index fedb7074..8ae2c1e0 100644 --- a/semconv/server.go +++ b/semconv/server.go @@ -2,11 +2,11 @@ package semconv var ( // ServerRequestDurationSeconds specifies meter metric name - ServerRequestDurationSeconds = "server_request_duration_seconds" + ServerRequestDurationSeconds = "micro_server_request_duration_seconds" // ServerRequestLatencyMicroseconds specifies meter metric name - ServerRequestLatencyMicroseconds = "server_request_latency_microseconds" + ServerRequestLatencyMicroseconds = "micro_server_request_latency_microseconds" // ServerRequestTotal specifies meter metric name - ServerRequestTotal = "server_request_total" + ServerRequestTotal = "micro_server_request_total" // ServerRequestInflight specifies meter metric name - ServerRequestInflight = "server_request_inflight" + ServerRequestInflight = "micro_server_request_inflight" ) diff --git a/semconv/store.go b/semconv/store.go index f3828a85..a9044f48 100644 --- a/semconv/store.go +++ b/semconv/store.go @@ -2,11 +2,11 @@ package semconv var ( // StoreRequestDurationSeconds specifies meter metric name - StoreRequestDurationSeconds = "store_request_duration_seconds" + StoreRequestDurationSeconds = "micro_store_request_duration_seconds" // ClientRequestLatencyMicroseconds specifies meter metric name - StoreRequestLatencyMicroseconds = "store_request_latency_microseconds" + StoreRequestLatencyMicroseconds = "micro_store_request_latency_microseconds" // StoreRequestTotal specifies meter metric name - StoreRequestTotal = "store_request_total" + StoreRequestTotal = "micro_store_request_total" // StoreRequestInflight specifies meter metric name - StoreRequestInflight = "store_request_inflight" + StoreRequestInflight = "micro_store_request_inflight" ) From faf2454f0a0659dc69eee2de0dc5044cc04c6166 Mon Sep 17 00:00:00 2001 From: Vasiliy Tolstov Date: Fri, 20 Sep 2024 17:54:17 +0300 Subject: [PATCH 40/59] cleanup Signed-off-by: Vasiliy Tolstov --- broker/broker.go | 2 +- client/client.go | 2 +- config/config.go | 2 +- errors/errors.go | 2 +- flow/flow.go | 2 +- fsm/fsm.go | 2 +- metadata/metadata.go | 2 +- meter/meter.go | 4 - meter/wrapper/wrapper.go | 347 --------------------- mtls/mtls.go | 2 +- network/network.go | 2 +- network/transport/transport.go | 2 +- network/tunnel/broker/broker.go | 2 +- network/tunnel/transport/transport.go | 2 +- network/tunnel/tunnel.go | 2 +- profiler/http/http.go | 2 +- profiler/pprof/pprof.go | 2 +- profiler/profile.go | 2 +- proxy/proxy.go | 2 +- register/register.go | 2 +- resolver/dns/dns.go | 2 +- resolver/dnssrv/dnssrv.go | 2 +- resolver/http/http.go | 2 +- resolver/noop/noop.go | 2 +- resolver/registry/registry.go | 2 +- resolver/static/static.go | 2 +- router/router.go | 2 +- selector/random/random.go | 2 +- selector/roundrobin/roundrobin.go | 2 +- selector/selector.go | 2 +- server/server.go | 6 +- store/store.go | 2 +- sync/sync.go | 2 +- tracer/tracer.go | 2 +- tracer/wrapper/wrapper.go | 415 -------------------------- util/addr/addr.go | 3 +- util/backoff/backoff.go | 2 +- util/buf/buf.go | 2 +- util/http/http.go | 2 +- util/id/id.go | 2 +- util/io/io.go | 2 +- util/jitter/random.go | 2 +- util/jitter/ticker.go | 2 +- util/net/net.go | 2 +- util/pki/pki.go | 2 +- util/pool/pool.go | 2 +- util/rand/rand.go | 2 +- util/register/util.go | 2 +- util/ring/buffer.go | 2 +- util/socket/socket.go | 2 +- util/stream/stream.go | 2 +- 51 files changed, 52 insertions(+), 815 deletions(-) delete mode 100644 meter/wrapper/wrapper.go delete mode 100644 tracer/wrapper/wrapper.go diff --git a/broker/broker.go b/broker/broker.go index 28098188..a67a0be8 100644 --- a/broker/broker.go +++ b/broker/broker.go @@ -1,5 +1,5 @@ // Package broker is an interface used for asynchronous messaging -package broker // import "go.unistack.org/micro/v3/broker" +package broker import ( "context" diff --git a/client/client.go b/client/client.go index e7c42692..5a0c4f7d 100644 --- a/client/client.go +++ b/client/client.go @@ -1,5 +1,5 @@ // Package client is an interface for an RPC client -package client // import "go.unistack.org/micro/v3/client" +package client import ( "context" diff --git a/config/config.go b/config/config.go index 955f2957..aa8ff16f 100644 --- a/config/config.go +++ b/config/config.go @@ -1,5 +1,5 @@ // Package config is an interface for dynamic configuration. -package config // import "go.unistack.org/micro/v3/config" +package config import ( "context" diff --git a/errors/errors.go b/errors/errors.go index 3224515d..c7db79e5 100644 --- a/errors/errors.go +++ b/errors/errors.go @@ -1,6 +1,6 @@ // Package errors provides a way to return detailed information // for an RPC request error. The error is normally JSON encoded. -package errors // import "go.unistack.org/micro/v3/errors" +package errors import ( "bytes" diff --git a/flow/flow.go b/flow/flow.go index 0124db3b..ed21b078 100644 --- a/flow/flow.go +++ b/flow/flow.go @@ -1,5 +1,5 @@ // Package flow is an interface used for saga pattern microservice workflow -package flow // import "go.unistack.org/micro/v3/flow" +package flow import ( "context" diff --git a/fsm/fsm.go b/fsm/fsm.go index 2cee6b1c..a4d072a6 100644 --- a/fsm/fsm.go +++ b/fsm/fsm.go @@ -1,4 +1,4 @@ -package fsm // import "go.unistack.org/micro/v3/fsm" +package fsm import ( "context" diff --git a/metadata/metadata.go b/metadata/metadata.go index bf7a8d3a..7fd26e8a 100644 --- a/metadata/metadata.go +++ b/metadata/metadata.go @@ -1,5 +1,5 @@ // Package metadata is a way of defining message headers -package metadata // import "go.unistack.org/micro/v3/metadata" +package metadata import ( "net/textproto" diff --git a/meter/meter.go b/meter/meter.go index 2afe65d5..e5d994fe 100644 --- a/meter/meter.go +++ b/meter/meter.go @@ -16,10 +16,6 @@ var ( DefaultAddress = ":9090" // DefaultPath the meter endpoint where the Meter data will be made available DefaultPath = "/metrics" - // DefaultMetricPrefix holds the string that prepends to all metrics - DefaultMetricPrefix = "micro_" - // DefaultLabelPrefix holds the string that prepends to all labels - DefaultLabelPrefix = "micro_" // DefaultSummaryQuantiles is the default spread of stats for summary DefaultSummaryQuantiles = []float64{0.5, 0.9, 0.97, 0.99, 1} // DefaultSummaryWindow is the default window for summary diff --git a/meter/wrapper/wrapper.go b/meter/wrapper/wrapper.go deleted file mode 100644 index 00ca68df..00000000 --- a/meter/wrapper/wrapper.go +++ /dev/null @@ -1,347 +0,0 @@ -package wrapper // import "go.unistack.org/micro/v3/meter/wrapper" - -import ( - "context" - "fmt" - "time" - - "go.unistack.org/micro/v3/client" - "go.unistack.org/micro/v3/meter" - "go.unistack.org/micro/v3/server" -) - -var ( - // ClientRequestDurationSeconds specifies meter metric name - ClientRequestDurationSeconds = "client_request_duration_seconds" - // ClientRequestLatencyMicroseconds specifies meter metric name - ClientRequestLatencyMicroseconds = "client_request_latency_microseconds" - // ClientRequestTotal specifies meter metric name - ClientRequestTotal = "client_request_total" - // ClientRequestInflight specifies meter metric name - ClientRequestInflight = "client_request_inflight" - // ServerRequestDurationSeconds specifies meter metric name - ServerRequestDurationSeconds = "server_request_duration_seconds" - // ServerRequestLatencyMicroseconds specifies meter metric name - ServerRequestLatencyMicroseconds = "server_request_latency_microseconds" - // ServerRequestTotal specifies meter metric name - ServerRequestTotal = "server_request_total" - // ServerRequestInflight specifies meter metric name - ServerRequestInflight = "server_request_inflight" - // PublishMessageDurationSeconds specifies meter metric name - PublishMessageDurationSeconds = "publish_message_duration_seconds" - // PublishMessageLatencyMicroseconds specifies meter metric name - PublishMessageLatencyMicroseconds = "publish_message_latency_microseconds" - // PublishMessageTotal specifies meter metric name - PublishMessageTotal = "publish_message_total" - // PublishMessageInflight specifies meter metric name - PublishMessageInflight = "publish_message_inflight" - // SubscribeMessageDurationSeconds specifies meter metric name - SubscribeMessageDurationSeconds = "subscribe_message_duration_seconds" - // SubscribeMessageLatencyMicroseconds specifies meter metric name - SubscribeMessageLatencyMicroseconds = "subscribe_message_latency_microseconds" - // SubscribeMessageTotal specifies meter metric name - SubscribeMessageTotal = "subscribe_message_total" - // SubscribeMessageInflight specifies meter metric name - SubscribeMessageInflight = "subscribe_message_inflight" - - labelSuccess = "success" - labelFailure = "failure" - labelStatus = "status" - labelEndpoint = "endpoint" - - // DefaultSkipEndpoints contains list of endpoints that not evaluted by wrapper - DefaultSkipEndpoints = []string{"Meter.Metrics", "Health.Live", "Health.Ready", "Health.Version"} -) - -// Options struct -type Options struct { - Meter meter.Meter - lopts []meter.Option - SkipEndpoints []string -} - -// Option func signature -type Option func(*Options) - -// NewOptions creates new Options struct -func NewOptions(opts ...Option) Options { - options := Options{ - Meter: meter.DefaultMeter, - lopts: make([]meter.Option, 0, 5), - SkipEndpoints: DefaultSkipEndpoints, - } - for _, o := range opts { - o(&options) - } - return options -} - -// ServiceName passes service name to meter label -func ServiceName(name string) Option { - return func(o *Options) { - o.lopts = append(o.lopts, meter.Labels("name", name)) - } -} - -// ServiceVersion passes service version to meter label -func ServiceVersion(version string) Option { - return func(o *Options) { - o.lopts = append(o.lopts, meter.Labels("version", version)) - } -} - -// ServiceID passes service id to meter label -func ServiceID(id string) Option { - return func(o *Options) { - o.lopts = append(o.lopts, meter.Labels("id", id)) - } -} - -// Meter passes meter -func Meter(m meter.Meter) Option { - return func(o *Options) { - o.Meter = m - } -} - -// SkipEndoints add endpoint to skip -func SkipEndoints(eps ...string) Option { - return func(o *Options) { - o.SkipEndpoints = append(o.SkipEndpoints, eps...) - } -} - -type wrapper struct { - client.Client - callFunc client.CallFunc - opts Options -} - -// NewClientWrapper create new client wrapper -func NewClientWrapper(opts ...Option) client.Wrapper { - return func(c client.Client) client.Client { - handler := &wrapper{ - opts: NewOptions(opts...), - Client: c, - } - return handler - } -} - -// NewCallWrapper create new call wrapper -func NewCallWrapper(opts ...Option) client.CallWrapper { - return func(fn client.CallFunc) client.CallFunc { - handler := &wrapper{ - opts: NewOptions(opts...), - callFunc: fn, - } - return handler.CallFunc - } -} - -func (w *wrapper) CallFunc(ctx context.Context, addr string, req client.Request, rsp interface{}, opts client.CallOptions) error { - endpoint := fmt.Sprintf("%s.%s", req.Service(), req.Endpoint()) - for _, ep := range w.opts.SkipEndpoints { - if ep == endpoint { - return w.callFunc(ctx, addr, req, rsp, opts) - } - } - - labels := make([]string, 0, 4) - labels = append(labels, labelEndpoint, endpoint) - - w.opts.Meter.Counter(ClientRequestInflight, labels...).Inc() - ts := time.Now() - err := w.callFunc(ctx, addr, req, rsp, opts) - te := time.Since(ts) - w.opts.Meter.Counter(ClientRequestInflight, labels...).Dec() - - w.opts.Meter.Summary(ClientRequestLatencyMicroseconds, labels...).Update(te.Seconds()) - w.opts.Meter.Histogram(ClientRequestDurationSeconds, labels...).Update(te.Seconds()) - - if err == nil { - labels = append(labels, labelStatus, labelSuccess) - } else { - labels = append(labels, labelStatus, labelFailure) - } - w.opts.Meter.Counter(ClientRequestTotal, labels...).Inc() - - return err -} - -func (w *wrapper) Call(ctx context.Context, req client.Request, rsp interface{}, opts ...client.CallOption) error { - endpoint := fmt.Sprintf("%s.%s", req.Service(), req.Endpoint()) - for _, ep := range w.opts.SkipEndpoints { - if ep == endpoint { - return w.Client.Call(ctx, req, rsp, opts...) - } - } - - labels := make([]string, 0, 4) - labels = append(labels, labelEndpoint, endpoint) - - w.opts.Meter.Counter(ClientRequestInflight, labels...).Inc() - ts := time.Now() - err := w.Client.Call(ctx, req, rsp, opts...) - te := time.Since(ts) - w.opts.Meter.Counter(ClientRequestInflight, labels...).Dec() - - w.opts.Meter.Summary(ClientRequestLatencyMicroseconds, labels...).Update(te.Seconds()) - w.opts.Meter.Histogram(ClientRequestDurationSeconds, labels...).Update(te.Seconds()) - - if err == nil { - labels = append(labels, labelStatus, labelSuccess) - } else { - labels = append(labels, labelStatus, labelFailure) - } - w.opts.Meter.Counter(ClientRequestTotal, labels...).Inc() - - return err -} - -func (w *wrapper) Stream(ctx context.Context, req client.Request, opts ...client.CallOption) (client.Stream, error) { - endpoint := fmt.Sprintf("%s.%s", req.Service(), req.Endpoint()) - for _, ep := range w.opts.SkipEndpoints { - if ep == endpoint { - return w.Client.Stream(ctx, req, opts...) - } - } - - labels := make([]string, 0, 4) - labels = append(labels, labelEndpoint, endpoint) - - w.opts.Meter.Counter(ClientRequestInflight, labels...).Inc() - ts := time.Now() - stream, err := w.Client.Stream(ctx, req, opts...) - te := time.Since(ts) - w.opts.Meter.Counter(ClientRequestInflight, labels...).Dec() - - w.opts.Meter.Summary(ClientRequestLatencyMicroseconds, labels...).Update(te.Seconds()) - w.opts.Meter.Histogram(ClientRequestDurationSeconds, labels...).Update(te.Seconds()) - - if err == nil { - labels = append(labels, labelStatus, labelSuccess) - } else { - labels = append(labels, labelStatus, labelFailure) - } - w.opts.Meter.Counter(ClientRequestTotal, labels...).Inc() - - return stream, err -} - -func (w *wrapper) Publish(ctx context.Context, p client.Message, opts ...client.PublishOption) error { - endpoint := p.Topic() - - labels := make([]string, 0, 4) - labels = append(labels, labelEndpoint, endpoint) - - w.opts.Meter.Counter(PublishMessageInflight, labels...).Inc() - ts := time.Now() - err := w.Client.Publish(ctx, p, opts...) - te := time.Since(ts) - w.opts.Meter.Counter(PublishMessageInflight, labels...).Dec() - - w.opts.Meter.Summary(PublishMessageLatencyMicroseconds, labels...).Update(te.Seconds()) - w.opts.Meter.Histogram(PublishMessageDurationSeconds, labels...).Update(te.Seconds()) - - if err == nil { - labels = append(labels, labelStatus, labelSuccess) - } else { - labels = append(labels, labelStatus, labelFailure) - } - w.opts.Meter.Counter(PublishMessageTotal, labels...).Inc() - - return err -} - -// NewHandlerWrapper create new server handler wrapper -// deprecated -func NewHandlerWrapper(opts ...Option) server.HandlerWrapper { - handler := &wrapper{ - opts: NewOptions(opts...), - } - return handler.HandlerFunc -} - -// NewServerHandlerWrapper create new server handler wrapper -func NewServerHandlerWrapper(opts ...Option) server.HandlerWrapper { - handler := &wrapper{ - opts: NewOptions(opts...), - } - return handler.HandlerFunc -} - -func (w *wrapper) HandlerFunc(fn server.HandlerFunc) server.HandlerFunc { - return func(ctx context.Context, req server.Request, rsp interface{}) error { - endpoint := req.Service() + "." + req.Endpoint() - for _, ep := range w.opts.SkipEndpoints { - if ep == endpoint { - return fn(ctx, req, rsp) - } - } - - labels := make([]string, 0, 4) - labels = append(labels, labelEndpoint, endpoint) - - w.opts.Meter.Counter(ServerRequestInflight, labels...).Inc() - ts := time.Now() - err := fn(ctx, req, rsp) - te := time.Since(ts) - w.opts.Meter.Counter(ServerRequestInflight, labels...).Dec() - - w.opts.Meter.Summary(ServerRequestLatencyMicroseconds, labels...).Update(te.Seconds()) - w.opts.Meter.Histogram(ServerRequestDurationSeconds, labels...).Update(te.Seconds()) - - if err == nil { - labels = append(labels, labelStatus, labelSuccess) - } else { - labels = append(labels, labelStatus, labelFailure) - } - w.opts.Meter.Counter(ServerRequestTotal, labels...).Inc() - - return err - } -} - -// NewSubscriberWrapper create server subscribe wrapper -// deprecated -func NewSubscriberWrapper(opts ...Option) server.SubscriberWrapper { - handler := &wrapper{ - opts: NewOptions(opts...), - } - return handler.SubscriberFunc -} - -func NewServerSubscriberWrapper(opts ...Option) server.SubscriberWrapper { - handler := &wrapper{ - opts: NewOptions(opts...), - } - return handler.SubscriberFunc -} - -func (w *wrapper) SubscriberFunc(fn server.SubscriberFunc) server.SubscriberFunc { - return func(ctx context.Context, msg server.Message) error { - endpoint := msg.Topic() - - labels := make([]string, 0, 4) - labels = append(labels, labelEndpoint, endpoint) - - w.opts.Meter.Counter(SubscribeMessageInflight, labels...).Inc() - ts := time.Now() - err := fn(ctx, msg) - te := time.Since(ts) - w.opts.Meter.Counter(SubscribeMessageInflight, labels...).Dec() - - w.opts.Meter.Summary(SubscribeMessageLatencyMicroseconds, labels...).Update(te.Seconds()) - w.opts.Meter.Histogram(SubscribeMessageDurationSeconds, labels...).Update(te.Seconds()) - - if err == nil { - labels = append(labels, labelStatus, labelSuccess) - } else { - labels = append(labels, labelStatus, labelFailure) - } - w.opts.Meter.Counter(SubscribeMessageTotal, labels...).Inc() - - return err - } -} diff --git a/mtls/mtls.go b/mtls/mtls.go index 238be41b..c0b1bc82 100644 --- a/mtls/mtls.go +++ b/mtls/mtls.go @@ -1,4 +1,4 @@ -package mtls // import "go.unistack.org/micro/v3/mtls" +package mtls import ( "bytes" diff --git a/network/network.go b/network/network.go index 9e00e9de..ec9bf96f 100644 --- a/network/network.go +++ b/network/network.go @@ -1,5 +1,5 @@ // Package network is for creating internetworks -package network // import "go.unistack.org/micro/v3/network" +package network import ( "go.unistack.org/micro/v3/client" diff --git a/network/transport/transport.go b/network/transport/transport.go index f70f34b9..2e33dcc5 100644 --- a/network/transport/transport.go +++ b/network/transport/transport.go @@ -1,5 +1,5 @@ // Package transport is an interface for synchronous connection based communication -package transport // import "go.unistack.org/micro/v3/network/transport" +package transport import ( "context" diff --git a/network/tunnel/broker/broker.go b/network/tunnel/broker/broker.go index 3066bd44..95a2f20f 100644 --- a/network/tunnel/broker/broker.go +++ b/network/tunnel/broker/broker.go @@ -1,5 +1,5 @@ // Package broker is a tunnel broker -package broker // import "go.unistack.org/micro/v3/network/tunnel/broker" +package broker import ( "context" diff --git a/network/tunnel/transport/transport.go b/network/tunnel/transport/transport.go index e5e6f68b..9ac71e2d 100644 --- a/network/tunnel/transport/transport.go +++ b/network/tunnel/transport/transport.go @@ -1,5 +1,5 @@ // Package transport provides a tunnel transport -package transport // import "go.unistack.org/micro/v3/network/tunnel/transport" +package transport import ( "context" diff --git a/network/tunnel/tunnel.go b/network/tunnel/tunnel.go index 3efa9294..e0d8620a 100644 --- a/network/tunnel/tunnel.go +++ b/network/tunnel/tunnel.go @@ -1,5 +1,5 @@ // Package tunnel provides gre network tunnelling -package tunnel // import "go.unistack.org/micro/v3/network/transport/tunnel" +package tunnel import ( "context" diff --git a/profiler/http/http.go b/profiler/http/http.go index d0fa4a80..b3dd7cb2 100644 --- a/profiler/http/http.go +++ b/profiler/http/http.go @@ -1,5 +1,5 @@ // Package http enables the http profiler -package http // import "go.unistack.org/micro/v3/profiler/http" +package http import ( "context" diff --git a/profiler/pprof/pprof.go b/profiler/pprof/pprof.go index 1c8fbdce..d96773e8 100644 --- a/profiler/pprof/pprof.go +++ b/profiler/pprof/pprof.go @@ -1,5 +1,5 @@ // Package pprof provides a pprof profiler which writes output to /tmp/[name].{cpu,mem}.pprof -package pprof // import "go.unistack.org/micro/v3/profiler/pprof" +package pprof import ( "os" diff --git a/profiler/profile.go b/profiler/profile.go index b29eb600..f14d5fb4 100644 --- a/profiler/profile.go +++ b/profiler/profile.go @@ -1,5 +1,5 @@ // Package profiler is for profilers -package profiler // import "go.unistack.org/micro/v3/profiler" +package profiler // Profiler interface type Profiler interface { diff --git a/proxy/proxy.go b/proxy/proxy.go index e80c1e9a..0a209e18 100644 --- a/proxy/proxy.go +++ b/proxy/proxy.go @@ -1,5 +1,5 @@ // Package proxy is a transparent proxy built on the micro/server -package proxy // import "go.unistack.org/micro/v3/proxy" +package proxy import ( "context" diff --git a/register/register.go b/register/register.go index bfe0078a..5f2bdab3 100644 --- a/register/register.go +++ b/register/register.go @@ -1,5 +1,5 @@ // Package register is an interface for service discovery -package register // import "go.unistack.org/micro/v3/register" +package register import ( "context" diff --git a/resolver/dns/dns.go b/resolver/dns/dns.go index 82de80d5..e80a9c8b 100644 --- a/resolver/dns/dns.go +++ b/resolver/dns/dns.go @@ -1,5 +1,5 @@ // Package dns resolves names to dns records -package dns // import "go.unistack.org/micro/v3/resolver/dns" +package dns import ( "context" diff --git a/resolver/dnssrv/dnssrv.go b/resolver/dnssrv/dnssrv.go index f88f816f..8b786824 100644 --- a/resolver/dnssrv/dnssrv.go +++ b/resolver/dnssrv/dnssrv.go @@ -1,5 +1,5 @@ // Package dnssrv resolves names to dns srv records -package dnssrv // import "go.unistack.org/micro/v3/resolver/dnssrv" +package dnssrv import ( "fmt" diff --git a/resolver/http/http.go b/resolver/http/http.go index 8aa8e43e..f32e791c 100644 --- a/resolver/http/http.go +++ b/resolver/http/http.go @@ -1,5 +1,5 @@ // Package http resolves names to network addresses using a http request -package http // import "go.unistack.org/micro/v3/resolver/http" +package http import ( "encoding/json" diff --git a/resolver/noop/noop.go b/resolver/noop/noop.go index 3fbe202e..71dfcb98 100644 --- a/resolver/noop/noop.go +++ b/resolver/noop/noop.go @@ -1,5 +1,5 @@ // Package noop is a noop resolver -package noop // import "go.unistack.org/micro/v3/resolver/noop" +package noop import ( "go.unistack.org/micro/v3/resolver" diff --git a/resolver/registry/registry.go b/resolver/registry/registry.go index 064f7b59..5f55e7a0 100644 --- a/resolver/registry/registry.go +++ b/resolver/registry/registry.go @@ -1,5 +1,5 @@ // Package register resolves names using the micro register -package register // import "go.unistack.org/micro/v3/resolver/registry" +package register import ( "context" diff --git a/resolver/static/static.go b/resolver/static/static.go index 4985a89f..9da00f5c 100644 --- a/resolver/static/static.go +++ b/resolver/static/static.go @@ -1,5 +1,5 @@ // Package static is a static resolver -package static // import "go.unistack.org/micro/v3/resolver/static" +package static import ( "go.unistack.org/micro/v3/resolver" diff --git a/router/router.go b/router/router.go index 43874196..8575d4ce 100644 --- a/router/router.go +++ b/router/router.go @@ -1,5 +1,5 @@ // Package router provides a network routing control plane -package router // import "go.unistack.org/micro/v3/router" +package router import ( "errors" diff --git a/selector/random/random.go b/selector/random/random.go index 40b25daf..2d739263 100644 --- a/selector/random/random.go +++ b/selector/random/random.go @@ -1,4 +1,4 @@ -package random // import "go.unistack.org/micro/v3/selector/random" +package random import ( "go.unistack.org/micro/v3/selector" diff --git a/selector/roundrobin/roundrobin.go b/selector/roundrobin/roundrobin.go index f39aca5c..f442b5b4 100644 --- a/selector/roundrobin/roundrobin.go +++ b/selector/roundrobin/roundrobin.go @@ -1,4 +1,4 @@ -package roundrobin // import "go.unistack.org/micro/v3/selector/roundrobin" +package roundrobin import ( "go.unistack.org/micro/v3/selector" diff --git a/selector/selector.go b/selector/selector.go index 2c16cca6..808481f9 100644 --- a/selector/selector.go +++ b/selector/selector.go @@ -1,5 +1,5 @@ // Package selector is for node selection and load balancing -package selector // import "go.unistack.org/micro/v3/selector" +package selector import ( "errors" diff --git a/server/server.go b/server/server.go index b6ada16a..49461e3a 100644 --- a/server/server.go +++ b/server/server.go @@ -1,5 +1,5 @@ // Package server is an interface for a micro server -package server // import "go.unistack.org/micro/v3/server" +package server import ( "context" @@ -11,7 +11,9 @@ import ( ) // DefaultServer default server -var DefaultServer Server = NewServer() +var ( + DefaultServer Server = NewServer() +) var ( // DefaultAddress will be used if no address passed, use secure localhost diff --git a/store/store.go b/store/store.go index 57646346..49eeff5c 100644 --- a/store/store.go +++ b/store/store.go @@ -1,5 +1,5 @@ // Package store is an interface for distributed data storage. -package store // import "go.unistack.org/micro/v3/store" +package store import ( "context" diff --git a/sync/sync.go b/sync/sync.go index 42329514..a84dfb64 100644 --- a/sync/sync.go +++ b/sync/sync.go @@ -1,5 +1,5 @@ // Package sync is an interface for distributed synchronization -package sync // import "go.unistack.org/micro/v3/sync" +package sync import ( "errors" diff --git a/tracer/tracer.go b/tracer/tracer.go index 8951da8a..e1cd2411 100644 --- a/tracer/tracer.go +++ b/tracer/tracer.go @@ -1,5 +1,5 @@ // Package tracer provides an interface for distributed tracing -package tracer // import "go.unistack.org/micro/v3/tracer" +package tracer import ( "context" diff --git a/tracer/wrapper/wrapper.go b/tracer/wrapper/wrapper.go deleted file mode 100644 index 0190c4e4..00000000 --- a/tracer/wrapper/wrapper.go +++ /dev/null @@ -1,415 +0,0 @@ -// Package wrapper provides wrapper for Tracer -package wrapper // import "go.unistack.org/micro/v3/tracer/wrapper" - -import ( - "context" - "fmt" - "strings" - - "go.unistack.org/micro/v3/client" - "go.unistack.org/micro/v3/metadata" - "go.unistack.org/micro/v3/server" - "go.unistack.org/micro/v3/tracer" -) - -var DefaultHeadersExctract = []string{metadata.HeaderXRequestID} - -func ExtractDefaultLabels(md metadata.Metadata) []interface{} { - labels := make([]interface{}, 0, len(DefaultHeadersExctract)) - for _, k := range DefaultHeadersExctract { - if v, ok := md.Get(k); ok { - labels = append(labels, strings.ToLower(k), v) - } - } - return labels -} - -var ( - DefaultClientCallObserver = func(ctx context.Context, req client.Request, rsp interface{}, opts []client.CallOption, sp tracer.Span, err error) { - var labels []interface{} - if md, ok := metadata.FromOutgoingContext(ctx); ok { - labels = append(labels, ExtractDefaultLabels(md)...) - } - if err != nil { - sp.SetStatus(tracer.SpanStatusError, err.Error()) - } - sp.AddLabels(labels...) - } - - DefaultClientStreamObserver = func(ctx context.Context, req client.Request, opts []client.CallOption, stream client.Stream, sp tracer.Span, err error) { - var labels []interface{} - if md, ok := metadata.FromOutgoingContext(ctx); ok { - labels = append(labels, ExtractDefaultLabels(md)...) - } - if err != nil { - sp.SetStatus(tracer.SpanStatusError, err.Error()) - } - sp.AddLabels(labels...) - } - - DefaultClientPublishObserver = func(ctx context.Context, msg client.Message, opts []client.PublishOption, sp tracer.Span, err error) { - var labels []interface{} - if md, ok := metadata.FromOutgoingContext(ctx); ok { - labels = append(labels, ExtractDefaultLabels(md)...) - } - labels = append(labels, ExtractDefaultLabels(msg.Metadata())...) - if err != nil { - sp.SetStatus(tracer.SpanStatusError, err.Error()) - } - sp.AddLabels(labels...) - } - - DefaultServerHandlerObserver = func(ctx context.Context, req server.Request, rsp interface{}, sp tracer.Span, err error) { - var labels []interface{} - if md, ok := metadata.FromIncomingContext(ctx); ok { - labels = append(labels, ExtractDefaultLabels(md)...) - } - if err != nil { - sp.SetStatus(tracer.SpanStatusError, err.Error()) - } - sp.AddLabels(labels...) - } - - DefaultServerSubscriberObserver = func(ctx context.Context, msg server.Message, sp tracer.Span, err error) { - var labels []interface{} - if md, ok := metadata.FromIncomingContext(ctx); ok { - labels = append(labels, ExtractDefaultLabels(md)...) - } - labels = append(labels, ExtractDefaultLabels(msg.Header())...) - - if err != nil { - sp.SetStatus(tracer.SpanStatusError, err.Error()) - } - sp.AddLabels(labels...) - } - - DefaultClientCallFuncObserver = func(ctx context.Context, addr string, req client.Request, rsp interface{}, opts client.CallOptions, sp tracer.Span, err error) { - sp.SetName(fmt.Sprintf("%s.%s call", req.Service(), req.Method())) - var labels []interface{} - if md, ok := metadata.FromOutgoingContext(ctx); ok { - labels = append(labels, ExtractDefaultLabels(md)...) - } - if err != nil { - sp.SetStatus(tracer.SpanStatusError, err.Error()) - } - sp.AddLabels(labels...) - } - - DefaultSkipEndpoints = []string{"Meter.Metrics", "Health.Live", "Health.Ready", "Health.Version"} -) - -type tWrapper struct { - client.Client - serverHandler server.HandlerFunc - serverSubscriber server.SubscriberFunc - clientCallFunc client.CallFunc - opts Options -} - -type ( - ClientCallObserver func(context.Context, client.Request, interface{}, []client.CallOption, tracer.Span, error) - ClientStreamObserver func(context.Context, client.Request, []client.CallOption, client.Stream, tracer.Span, error) - ClientPublishObserver func(context.Context, client.Message, []client.PublishOption, tracer.Span, error) - ClientCallFuncObserver func(context.Context, string, client.Request, interface{}, client.CallOptions, tracer.Span, error) - ServerHandlerObserver func(context.Context, server.Request, interface{}, tracer.Span, error) - ServerSubscriberObserver func(context.Context, server.Message, tracer.Span, error) -) - -// Options struct -type Options struct { - // Tracer that used for tracing - Tracer tracer.Tracer - // ClientCallObservers funcs - ClientCallObservers []ClientCallObserver - // ClientStreamObservers funcs - ClientStreamObservers []ClientStreamObserver - // ClientPublishObservers funcs - ClientPublishObservers []ClientPublishObserver - // ClientCallFuncObservers funcs - ClientCallFuncObservers []ClientCallFuncObserver - // ServerHandlerObservers funcs - ServerHandlerObservers []ServerHandlerObserver - // ServerSubscriberObservers funcs - ServerSubscriberObservers []ServerSubscriberObserver - // SkipEndpoints - SkipEndpoints []string -} - -// Option func signature -type Option func(*Options) - -// NewOptions create Options from Option slice -func NewOptions(opts ...Option) Options { - options := Options{ - Tracer: tracer.DefaultTracer, - ClientCallObservers: []ClientCallObserver{DefaultClientCallObserver}, - ClientStreamObservers: []ClientStreamObserver{DefaultClientStreamObserver}, - ClientPublishObservers: []ClientPublishObserver{DefaultClientPublishObserver}, - ClientCallFuncObservers: []ClientCallFuncObserver{DefaultClientCallFuncObserver}, - ServerHandlerObservers: []ServerHandlerObserver{DefaultServerHandlerObserver}, - ServerSubscriberObservers: []ServerSubscriberObserver{DefaultServerSubscriberObserver}, - SkipEndpoints: DefaultSkipEndpoints, - } - - for _, o := range opts { - o(&options) - } - - return options -} - -// WithTracer pass tracer -func WithTracer(t tracer.Tracer) Option { - return func(o *Options) { - o.Tracer = t - } -} - -// SkipEndponts -func SkipEndpoins(eps ...string) Option { - return func(o *Options) { - o.SkipEndpoints = append(o.SkipEndpoints, eps...) - } -} - -// WithClientCallObservers funcs -func WithClientCallObservers(ob ...ClientCallObserver) Option { - return func(o *Options) { - o.ClientCallObservers = ob - } -} - -// WithClientStreamObservers funcs -func WithClientStreamObservers(ob ...ClientStreamObserver) Option { - return func(o *Options) { - o.ClientStreamObservers = ob - } -} - -// WithClientPublishObservers funcs -func WithClientPublishObservers(ob ...ClientPublishObserver) Option { - return func(o *Options) { - o.ClientPublishObservers = ob - } -} - -// WithClientCallFuncObservers funcs -func WithClientCallFuncObservers(ob ...ClientCallFuncObserver) Option { - return func(o *Options) { - o.ClientCallFuncObservers = ob - } -} - -// WithServerHandlerObservers funcs -func WithServerHandlerObservers(ob ...ServerHandlerObserver) Option { - return func(o *Options) { - o.ServerHandlerObservers = ob - } -} - -// WithServerSubscriberObservers funcs -func WithServerSubscriberObservers(ob ...ServerSubscriberObserver) Option { - return func(o *Options) { - o.ServerSubscriberObservers = ob - } -} - -func (ot *tWrapper) Call(ctx context.Context, req client.Request, rsp interface{}, opts ...client.CallOption) error { - endpoint := fmt.Sprintf("%s.%s", req.Service(), req.Endpoint()) - for _, ep := range ot.opts.SkipEndpoints { - if ep == endpoint { - return ot.Client.Call(ctx, req, rsp, opts...) - } - } - - nctx, sp := ot.opts.Tracer.Start(ctx, fmt.Sprintf("%s.%s rpc-client", req.Service(), req.Method()), - tracer.WithSpanKind(tracer.SpanKindClient), - tracer.WithSpanLabels( - "rpc.service", req.Service(), - "rpc.method", req.Method(), - "rpc.flavor", "rpc", - "rpc.call", "/"+req.Service()+"/"+req.Endpoint(), - "rpc.call_type", "unary", - ), - ) - defer sp.Finish() - - err := ot.Client.Call(nctx, req, rsp, opts...) - - for _, o := range ot.opts.ClientCallObservers { - o(nctx, req, rsp, opts, sp, err) - } - - return err -} - -func (ot *tWrapper) Stream(ctx context.Context, req client.Request, opts ...client.CallOption) (client.Stream, error) { - endpoint := fmt.Sprintf("%s.%s", req.Service(), req.Endpoint()) - for _, ep := range ot.opts.SkipEndpoints { - if ep == endpoint { - return ot.Client.Stream(ctx, req, opts...) - } - } - - nctx, sp := ot.opts.Tracer.Start(ctx, fmt.Sprintf("%s.%s rpc-client", req.Service(), req.Method()), - tracer.WithSpanKind(tracer.SpanKindClient), - tracer.WithSpanLabels( - "rpc.service", req.Service(), - "rpc.method", req.Method(), - "rpc.flavor", "rpc", - "rpc.call", "/"+req.Service()+"/"+req.Endpoint(), - "rpc.call_type", "stream", - ), - ) - defer sp.Finish() - - stream, err := ot.Client.Stream(nctx, req, opts...) - - for _, o := range ot.opts.ClientStreamObservers { - o(nctx, req, opts, stream, sp, err) - } - - return stream, err -} - -func (ot *tWrapper) Publish(ctx context.Context, msg client.Message, opts ...client.PublishOption) error { - nctx, sp := ot.opts.Tracer.Start(ctx, msg.Topic()+" publish", tracer.WithSpanKind(tracer.SpanKindProducer)) - defer sp.Finish() - sp.AddLabels("messaging.destination.name", msg.Topic()) - sp.AddLabels("messaging.operation", "publish") - err := ot.Client.Publish(nctx, msg, opts...) - - for _, o := range ot.opts.ClientPublishObservers { - o(nctx, msg, opts, sp, err) - } - - return err -} - -func (ot *tWrapper) ServerHandler(ctx context.Context, req server.Request, rsp interface{}) error { - endpoint := fmt.Sprintf("%s.%s", req.Service(), req.Method()) - for _, ep := range ot.opts.SkipEndpoints { - if ep == endpoint { - return ot.serverHandler(ctx, req, rsp) - } - } - - callType := "unary" - if req.Stream() { - callType = "stream" - } - - nctx, sp := ot.opts.Tracer.Start(ctx, fmt.Sprintf("%s.%s rpc-server", req.Service(), req.Method()), - tracer.WithSpanKind(tracer.SpanKindServer), - tracer.WithSpanLabels( - "rpc.service", req.Service(), - "rpc.method", req.Method(), - "rpc.flavor", "rpc", - "rpc.call", "/"+req.Service()+"/"+req.Endpoint(), - "rpc.call_type", callType, - ), - ) - defer sp.Finish() - - err := ot.serverHandler(nctx, req, rsp) - - for _, o := range ot.opts.ServerHandlerObservers { - o(nctx, req, rsp, sp, err) - } - - return err -} - -func (ot *tWrapper) ServerSubscriber(ctx context.Context, msg server.Message) error { - nctx, sp := ot.opts.Tracer.Start(ctx, msg.Topic()+" process", tracer.WithSpanKind(tracer.SpanKindConsumer)) - defer sp.Finish() - sp.AddLabels("messaging.operation", "process") - sp.AddLabels("messaging.source.name", msg.Topic()) - err := ot.serverSubscriber(nctx, msg) - - for _, o := range ot.opts.ServerSubscriberObservers { - o(nctx, msg, sp, err) - } - - return err -} - -// NewClientWrapper accepts an open tracing Trace and returns a Client Wrapper -func NewClientWrapper(opts ...Option) client.Wrapper { - return func(c client.Client) client.Client { - options := NewOptions() - for _, o := range opts { - o(&options) - } - return &tWrapper{opts: options, Client: c} - } -} - -// NewClientCallWrapper accepts an opentracing Tracer and returns a Call Wrapper -func NewClientCallWrapper(opts ...Option) client.CallWrapper { - return func(h client.CallFunc) client.CallFunc { - options := NewOptions() - for _, o := range opts { - o(&options) - } - - ot := &tWrapper{opts: options, clientCallFunc: h} - return ot.ClientCallFunc - } -} - -func (ot *tWrapper) ClientCallFunc(ctx context.Context, addr string, req client.Request, rsp interface{}, opts client.CallOptions) error { - endpoint := fmt.Sprintf("%s.%s", req.Service(), req.Method()) - for _, ep := range ot.opts.SkipEndpoints { - if ep == endpoint { - return ot.ClientCallFunc(ctx, addr, req, rsp, opts) - } - } - - nctx, sp := ot.opts.Tracer.Start(ctx, fmt.Sprintf("%s.%s rpc-client", req.Service(), req.Method()), - tracer.WithSpanKind(tracer.SpanKindClient), - tracer.WithSpanLabels( - "rpc.service", req.Service(), - "rpc.method", req.Method(), - "rpc.flavor", "rpc", - "rpc.call", "/"+req.Service()+"/"+req.Endpoint(), - "rpc.call_type", "unary", - ), - ) - - defer sp.Finish() - - err := ot.clientCallFunc(nctx, addr, req, rsp, opts) - - for _, o := range ot.opts.ClientCallFuncObservers { - o(nctx, addr, req, rsp, opts, sp, err) - } - - return err -} - -// NewServerHandlerWrapper accepts an options and returns a Handler Wrapper -func NewServerHandlerWrapper(opts ...Option) server.HandlerWrapper { - return func(h server.HandlerFunc) server.HandlerFunc { - options := NewOptions() - for _, o := range opts { - o(&options) - } - - ot := &tWrapper{opts: options, serverHandler: h} - return ot.ServerHandler - } -} - -// NewServerSubscriberWrapper accepts an opentracing Tracer and returns a Subscriber Wrapper -func NewServerSubscriberWrapper(opts ...Option) server.SubscriberWrapper { - return func(h server.SubscriberFunc) server.SubscriberFunc { - options := NewOptions() - for _, o := range opts { - o(&options) - } - - ot := &tWrapper{opts: options, serverSubscriber: h} - return ot.ServerSubscriber - } -} diff --git a/util/addr/addr.go b/util/addr/addr.go index 47e5636f..9415ec94 100644 --- a/util/addr/addr.go +++ b/util/addr/addr.go @@ -1,4 +1,4 @@ -package addr // import "go.unistack.org/micro/v3/util/addr" +package addr import ( "fmt" @@ -58,6 +58,7 @@ func IsLocal(addr string) bool { } // Extract returns a real ip +// //nolint:gocyclo func Extract(addr string) (string, error) { // if addr specified then its returned diff --git a/util/backoff/backoff.go b/util/backoff/backoff.go index a18c6a52..cadc2fc1 100644 --- a/util/backoff/backoff.go +++ b/util/backoff/backoff.go @@ -1,5 +1,5 @@ // Package backoff provides backoff functionality -package backoff // import "go.unistack.org/micro/v3/util/backoff" +package backoff import ( "math" diff --git a/util/buf/buf.go b/util/buf/buf.go index f01a9d6a..e6b86214 100644 --- a/util/buf/buf.go +++ b/util/buf/buf.go @@ -1,4 +1,4 @@ -package buf // import "go.unistack.org/micro/v3/util/buf" +package buf import ( "bytes" diff --git a/util/http/http.go b/util/http/http.go index de83f138..571effbf 100644 --- a/util/http/http.go +++ b/util/http/http.go @@ -1,4 +1,4 @@ -package http // import "go.unistack.org/micro/v3/util/http" +package http import ( "context" diff --git a/util/id/id.go b/util/id/id.go index f4699e6e..2d094f1f 100644 --- a/util/id/id.go +++ b/util/id/id.go @@ -1,4 +1,4 @@ -package id // import "go.unistack.org/micro/v3/util/id" +package id import ( "context" diff --git a/util/io/io.go b/util/io/io.go index 54599523..ed4d8fc0 100644 --- a/util/io/io.go +++ b/util/io/io.go @@ -1,5 +1,5 @@ // Package io is for io management -package io // import "go.unistack.org/micro/v3/util/io" +package io import ( "io" diff --git a/util/jitter/random.go b/util/jitter/random.go index eb03b29f..8e1cae69 100644 --- a/util/jitter/random.go +++ b/util/jitter/random.go @@ -1,5 +1,5 @@ // Package jitter provides a random jitter -package jitter // import "go.unistack.org/micro/v3/util/jitter" +package jitter import ( "time" diff --git a/util/jitter/ticker.go b/util/jitter/ticker.go index 84946556..b8d06ee5 100644 --- a/util/jitter/ticker.go +++ b/util/jitter/ticker.go @@ -1,4 +1,4 @@ -package jitter // import "go.unistack.org/micro/v3/util/jitter" +package jitter import ( "context" diff --git a/util/net/net.go b/util/net/net.go index a5e8e278..96a4f1e7 100644 --- a/util/net/net.go +++ b/util/net/net.go @@ -1,4 +1,4 @@ -package net // import "go.unistack.org/micro/v3/util/net" +package net import ( "errors" diff --git a/util/pki/pki.go b/util/pki/pki.go index 681821bf..e09cab82 100644 --- a/util/pki/pki.go +++ b/util/pki/pki.go @@ -1,6 +1,6 @@ // Package pki provides PKI all the PKI functions necessary to run micro over an untrusted network // including a CA -package pki // import "go.unistack.org/micro/v3/util/pki" +package pki import ( "bytes" diff --git a/util/pool/pool.go b/util/pool/pool.go index 576d1873..2433cd59 100644 --- a/util/pool/pool.go +++ b/util/pool/pool.go @@ -1,5 +1,5 @@ // Package pool is a connection pool -package pool // import "go.unistack.org/micro/v3/util/pool" +package pool import ( "context" diff --git a/util/rand/rand.go b/util/rand/rand.go index 3df9701c..a6243365 100644 --- a/util/rand/rand.go +++ b/util/rand/rand.go @@ -1,4 +1,4 @@ -package rand // import "go.unistack.org/micro/v3/util/rand" +package rand import ( crand "crypto/rand" diff --git a/util/register/util.go b/util/register/util.go index e3867389..9bb8b48a 100644 --- a/util/register/util.go +++ b/util/register/util.go @@ -1,4 +1,4 @@ -package register // import "go.unistack.org/micro/v3/util/register" +package register import ( "context" diff --git a/util/ring/buffer.go b/util/ring/buffer.go index 50fe2499..9cc4b891 100644 --- a/util/ring/buffer.go +++ b/util/ring/buffer.go @@ -1,5 +1,5 @@ // Package ring provides a simple ring buffer for storing local data -package ring // import "go.unistack.org/micro/v3/util/ring" +package ring import ( "sync" diff --git a/util/socket/socket.go b/util/socket/socket.go index 1797a71a..0ae12b4a 100644 --- a/util/socket/socket.go +++ b/util/socket/socket.go @@ -1,5 +1,5 @@ // Package socket provides a pseudo socket -package socket // import "go.unistack.org/micro/v3/util/socket" +package socket import ( "io" diff --git a/util/stream/stream.go b/util/stream/stream.go index 5e51df98..36bf06b1 100644 --- a/util/stream/stream.go +++ b/util/stream/stream.go @@ -1,5 +1,5 @@ // Package stream encapsulates streams within streams -package stream // import "go.unistack.org/micro/v3/util/stream" +package stream import ( "context" From 6641463eed5238df45db7839ee98eb0f03d8056b Mon Sep 17 00:00:00 2001 From: Vasiliy Tolstov Date: Fri, 20 Sep 2024 19:22:20 +0300 Subject: [PATCH 41/59] util/reflect: add ability to merge maps Signed-off-by: Vasiliy Tolstov --- util/reflect/reflect.go | 36 ++++++++++++++++++++++++++++++++++++ util/reflect/reflect_test.go | 21 +++++++++++++++++++++ 2 files changed, 57 insertions(+) diff --git a/util/reflect/reflect.go b/util/reflect/reflect.go index f0cda32c..729b4e02 100644 --- a/util/reflect/reflect.go +++ b/util/reflect/reflect.go @@ -44,6 +44,37 @@ func SliceAppend(b bool) Option { } } +var maxDepth = 32 + +func mergeMap(dst, src map[string]interface{}, depth int) map[string]interface{} { + if depth > maxDepth { + return dst + } + for key, srcVal := range src { + if dstVal, ok := dst[key]; ok { + srcMap, srcMapOk := mapify(srcVal) + dstMap, dstMapOk := mapify(dstVal) + if srcMapOk && dstMapOk { + srcVal = mergeMap(dstMap, srcMap, depth+1) + } + } + dst[key] = srcVal + } + return dst +} + +func mapify(i interface{}) (map[string]interface{}, bool) { + value := reflect.ValueOf(i) + if value.Kind() == reflect.Map { + m := map[string]interface{}{} + for _, k := range value.MapKeys() { + m[k.String()] = value.MapIndex(k).Interface() + } + return m, true + } + return map[string]interface{}{}, false +} + // Merge merges map[string]interface{} to destination struct func Merge(dst interface{}, mp map[string]interface{}, opts ...Option) error { options := Options{} @@ -59,6 +90,11 @@ func Merge(dst interface{}, mp map[string]interface{}, opts ...Option) error { return err } + if mapper, ok := dst.(map[string]interface{}); ok { + dst = mergeMap(mapper, mp, 0) + return nil + } + var err error var sval reflect.Value var fname string diff --git a/util/reflect/reflect_test.go b/util/reflect/reflect_test.go index a2cef0de..c11d7736 100644 --- a/util/reflect/reflect_test.go +++ b/util/reflect/reflect_test.go @@ -4,6 +4,27 @@ import ( "testing" ) +func TestMergeMap(t *testing.T) { + src := map[string]interface{}{ + "skey1": "sval1", + "skey2": map[string]interface{}{ + "skey3": "sval3", + }, + } + dst := map[string]interface{}{ + "skey1": "dval1", + "skey2": map[string]interface{}{ + "skey3": "dval3", + }, + } + + if err := Merge(src, dst); err != nil { + t.Fatal(err) + } + + t.Logf("%#+v", src) +} + func TestFieldName(t *testing.T) { src := "SomeVar" chk := "some_var" From 82d269cfb451e5394faf8a733986a44b02a6e9b5 Mon Sep 17 00:00:00 2001 From: Vasiliy Tolstov Date: Sun, 29 Sep 2024 22:58:53 +0300 Subject: [PATCH 42/59] xpool: add metrics Signed-off-by: Vasiliy Tolstov --- meter/meter.go | 2 + util/xpool/pool.go | 217 ++++++++++++++++++++++++++++++++++++++-- util/xpool/pool_test.go | 24 ++++- 3 files changed, 232 insertions(+), 11 deletions(-) diff --git a/meter/meter.go b/meter/meter.go index e5d994fe..42b20473 100644 --- a/meter/meter.go +++ b/meter/meter.go @@ -16,6 +16,8 @@ var ( DefaultAddress = ":9090" // DefaultPath the meter endpoint where the Meter data will be made available DefaultPath = "/metrics" + // DefaultMeterStatsInterval specifies interval for meter updating + DefaultMeterStatsInterval = 5 * time.Second // DefaultSummaryQuantiles is the default spread of stats for summary DefaultSummaryQuantiles = []float64{0.5, 0.9, 0.97, 0.99, 1} // DefaultSummaryWindow is the default window for summary diff --git a/util/xpool/pool.go b/util/xpool/pool.go index fa27df3a..b80cd482 100644 --- a/util/xpool/pool.go +++ b/util/xpool/pool.go @@ -2,13 +2,77 @@ package pool import ( "bytes" + "strconv" + "strings" "sync" + "sync/atomic" + "time" + + "go.unistack.org/micro/v3/meter" + "go.unistack.org/micro/v3/semconv" +) + +var ( + pools = make([]Statser, 0) + poolsMu sync.Mutex +) + +// Stats struct +type Stats struct { + Get uint64 + Put uint64 + Mis uint64 + Ret uint64 +} + +// Statser provides buffer pool stats +type Statser interface { + Stats() Stats + Cap() int +} + +func init() { + go newStatsMeter() +} + +func newStatsMeter() { + ticker := time.NewTicker(meter.DefaultMeterStatsInterval) + defer ticker.Stop() + + for { + select { + case <-ticker.C: + poolsMu.Lock() + for _, st := range pools { + stats := st.Stats() + meter.DefaultMeter.Counter(semconv.PoolGetTotal, "capacity", strconv.Itoa(st.Cap())).Set(stats.Get) + meter.DefaultMeter.Counter(semconv.PoolPutTotal, "capacity", strconv.Itoa(st.Cap())).Set(stats.Put) + meter.DefaultMeter.Counter(semconv.PoolMisTotal, "capacity", strconv.Itoa(st.Cap())).Set(stats.Mis) + meter.DefaultMeter.Counter(semconv.PoolRetTotal, "capacity", strconv.Itoa(st.Cap())).Set(stats.Ret) + } + poolsMu.Unlock() + } + } +} + +var ( + _ Statser = (*BytePool)(nil) + _ Statser = (*BytesPool)(nil) + _ Statser = (*StringsPool)(nil) ) type Pool[T any] struct { p *sync.Pool } +func (p Pool[T]) Put(t T) { + p.p.Put(t) +} + +func (p Pool[T]) Get() T { + return p.p.Get().(T) +} + func NewPool[T any](fn func() T) Pool[T] { return Pool[T]{ p: &sync.Pool{ @@ -19,18 +83,155 @@ func NewPool[T any](fn func() T) Pool[T] { } } -func (p Pool[T]) Get() T { - return p.p.Get().(T) +type BytePool struct { + p *sync.Pool + get uint64 + put uint64 + mis uint64 + ret uint64 + c int } -func (p Pool[T]) Put(t T) { - p.p.Put(t) +func NewBytePool(size int) *BytePool { + p := &BytePool{c: size} + p.p = &sync.Pool{ + New: func() interface{} { + atomic.AddUint64(&p.mis, 1) + b := make([]byte, 0, size) + return &b + }, + } + poolsMu.Lock() + pools = append(pools, p) + poolsMu.Unlock() + return p } -func NewBytePool(size int) Pool[[]byte] { - return NewPool(func() []byte { return make([]byte, size) }) +func (p *BytePool) Cap() int { + return p.c } -func NewBytesPool() Pool[*bytes.Buffer] { - return NewPool(func() *bytes.Buffer { return bytes.NewBuffer(nil) }) +func (p *BytePool) Stats() Stats { + return Stats{ + Put: atomic.LoadUint64(&p.put), + Get: atomic.LoadUint64(&p.get), + Mis: atomic.LoadUint64(&p.mis), + Ret: atomic.LoadUint64(&p.ret), + } +} + +func (p *BytePool) Get() *[]byte { + atomic.AddUint64(&p.get, 1) + return p.p.Get().(*[]byte) +} + +func (p *BytePool) Put(b *[]byte) { + atomic.AddUint64(&p.put, 1) + if cap(*b) > p.c { + atomic.AddUint64(&p.ret, 1) + return + } + *b = (*b)[:0] + p.p.Put(b) +} + +type BytesPool struct { + p *sync.Pool + get uint64 + put uint64 + mis uint64 + ret uint64 + c int +} + +func NewBytesPool(size int) *BytesPool { + p := &BytesPool{c: size} + p.p = &sync.Pool{ + New: func() interface{} { + atomic.AddUint64(&p.mis, 1) + b := bytes.NewBuffer(make([]byte, 0, size)) + return b + }, + } + poolsMu.Lock() + pools = append(pools, p) + poolsMu.Unlock() + return p +} + +func (p *BytesPool) Cap() int { + return p.c +} + +func (p *BytesPool) Stats() Stats { + return Stats{ + Put: atomic.LoadUint64(&p.put), + Get: atomic.LoadUint64(&p.get), + Mis: atomic.LoadUint64(&p.mis), + Ret: atomic.LoadUint64(&p.ret), + } +} + +func (p *BytesPool) Get() *bytes.Buffer { + return p.p.Get().(*bytes.Buffer) +} + +func (p *BytesPool) Put(b *bytes.Buffer) { + if (*b).Cap() > p.c { + atomic.AddUint64(&p.ret, 1) + return + } + b.Reset() + p.p.Put(b) +} + +type StringsPool struct { + p *sync.Pool + get uint64 + put uint64 + mis uint64 + ret uint64 + c int +} + +func NewStringsPool(size int) *StringsPool { + p := &StringsPool{c: size} + p.p = &sync.Pool{ + New: func() interface{} { + atomic.AddUint64(&p.mis, 1) + return &strings.Builder{} + }, + } + poolsMu.Lock() + pools = append(pools, p) + poolsMu.Unlock() + return p +} + +func (p *StringsPool) Cap() int { + return p.c +} + +func (p *StringsPool) Stats() Stats { + return Stats{ + Put: atomic.LoadUint64(&p.put), + Get: atomic.LoadUint64(&p.get), + Mis: atomic.LoadUint64(&p.mis), + Ret: atomic.LoadUint64(&p.ret), + } +} + +func (p *StringsPool) Get() *strings.Builder { + atomic.AddUint64(&p.get, 1) + return p.p.Get().(*strings.Builder) +} + +func (p *StringsPool) Put(b *strings.Builder) { + atomic.AddUint64(&p.put, 1) + if b.Cap() > p.c { + atomic.AddUint64(&p.ret, 1) + return + } + b.Reset() + p.p.Put(b) } diff --git a/util/xpool/pool_test.go b/util/xpool/pool_test.go index 8e7a9b81..710a0024 100644 --- a/util/xpool/pool_test.go +++ b/util/xpool/pool_test.go @@ -2,12 +2,30 @@ package pool import ( "bytes" - "strings" "testing" ) +func TestByte(t *testing.T) { + p := NewBytePool(1024) + b := p.Get() + copy(*b, []byte(`test`)) + if bytes.Equal(*b, []byte("test")) { + t.Fatal("pool not works") + } + p.Put(b) + b = p.Get() + for i := 0; i < 1500; i++ { + *b = append(*b, []byte(`test`)...) + } + p.Put(b) + st := p.Stats() + if st.Get != 2 && st.Put != 2 && st.Mis != 1 && st.Ret != 1 { + t.Fatalf("pool stats error %#+v", st) + } +} + func TestBytes(t *testing.T) { - p := NewPool(func() *bytes.Buffer { return bytes.NewBuffer(nil) }) + p := NewBytesPool(1024) b := p.Get() b.Write([]byte(`test`)) if b.String() != "test" { @@ -17,7 +35,7 @@ func TestBytes(t *testing.T) { } func TestStrings(t *testing.T) { - p := NewPool(func() *strings.Builder { return &strings.Builder{} }) + p := NewStringsPool(20) b := p.Get() b.Write([]byte(`test`)) if b.String() != "test" { From fd2b2762e9877eb0250307cf1b87d96bd52233c7 Mon Sep 17 00:00:00 2001 From: Vasiliy Tolstov Date: Mon, 30 Sep 2024 09:57:07 +0300 Subject: [PATCH 43/59] fixup missing xpool dep Signed-off-by: Vasiliy Tolstov --- semconv/pool.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) create mode 100644 semconv/pool.go diff --git a/semconv/pool.go b/semconv/pool.go new file mode 100644 index 00000000..b32c4a4d --- /dev/null +++ b/semconv/pool.go @@ -0,0 +1,12 @@ +package semconv + +var ( + // PoolGetTotal specifies meter metric name for total number of pool get ops + PoolGetTotal = "micro_pool_get_total" + // PoolPutTotal specifies meter metric name for total number of pool put ops + PoolPutTotal = "micro_pool_put_total" + // PoolMisTotal specifies meter metric name for total number of pool misses + PoolMisTotal = "micro_pool_mis_total" + // PoolRetTotal specifies meter metric name for total number of pool returned to gc + PoolRetTotal = "micro_pool_ret_total" +) From f1b8ecbdb3febe6deaeadf39af984f8d6e7a78c6 Mon Sep 17 00:00:00 2001 From: Vasiliy Tolstov Date: Sat, 5 Oct 2024 14:46:22 +0300 Subject: [PATCH 44/59] store: add new ErrNotConnected error Signed-off-by: Vasiliy Tolstov --- store/store.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/store/store.go b/store/store.go index 49eeff5c..25530cc4 100644 --- a/store/store.go +++ b/store/store.go @@ -7,6 +7,8 @@ import ( ) var ( + // ErrNotConnected is returned when a store is not connected + ErrNotConnected = errors.New("not conected") // ErrNotFound is returned when a key doesn't exist ErrNotFound = errors.New("not found") // ErrInvalidKey is returned when a key has empty or have invalid format From 71fe0df73f20cddb3cc36f797d2977b6091f4af1 Mon Sep 17 00:00:00 2001 From: Vasiliy Tolstov Date: Sun, 6 Oct 2024 13:50:59 +0300 Subject: [PATCH 45/59] use automaxproc and automemlimit Signed-off-by: Vasiliy Tolstov --- go.mod | 16 ++++++++++++++-- go.sum | 49 +++++++++++++++++++++++++++++++++++++++++++++++-- service.go | 14 ++++++++++++++ 3 files changed, 75 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 5dd808cb..9052eded 100644 --- a/go.mod +++ b/go.mod @@ -1,20 +1,32 @@ module go.unistack.org/micro/v3 -go 1.20 +go 1.22 require ( dario.cat/mergo v1.0.0 github.com/DATA-DOG/go-sqlmock v1.5.0 + github.com/KimMachineGun/automemlimit v0.6.1 github.com/google/uuid v1.3.0 github.com/patrickmn/go-cache v2.1.0+incompatible github.com/silas/dag v0.0.0-20220518035006-a7e85ada93c5 + go.uber.org/automaxprocs v1.6.0 + go.unistack.org/micro-proto/v3 v3.4.1 golang.org/x/sync v0.3.0 google.golang.org/grpc v1.57.0 - google.golang.org/protobuf v1.31.0 + google.golang.org/protobuf v1.33.0 ) require ( + github.com/cilium/ebpf v0.9.1 // indirect + github.com/containerd/cgroups/v3 v3.0.1 // indirect + github.com/coreos/go-systemd/v22 v22.3.2 // indirect + github.com/docker/go-units v0.4.0 // indirect + github.com/godbus/dbus/v5 v5.0.4 // indirect github.com/golang/protobuf v1.5.3 // indirect + github.com/opencontainers/runtime-spec v1.0.2 // indirect + github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect + github.com/sirupsen/logrus v1.8.1 // indirect golang.org/x/net v0.14.0 // indirect + golang.org/x/sys v0.11.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20230526203410-71b5a4ffd15e // indirect ) diff --git a/go.sum b/go.sum index 5f3145b9..025a4af7 100644 --- a/go.sum +++ b/go.sum @@ -2,23 +2,68 @@ dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk= dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk= github.com/DATA-DOG/go-sqlmock v1.5.0 h1:Shsta01QNfFxHCfpW6YH2STWB0MudeXXEWMr20OEh60= github.com/DATA-DOG/go-sqlmock v1.5.0/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= +github.com/KimMachineGun/automemlimit v0.6.1 h1:ILa9j1onAAMadBsyyUJv5cack8Y1WT26yLj/V+ulKp8= +github.com/KimMachineGun/automemlimit v0.6.1/go.mod h1:T7xYht7B8r6AG/AqFcUdc7fzd2bIdBKmepfP2S1svPY= +github.com/cilium/ebpf v0.9.1 h1:64sn2K3UKw8NbP/blsixRpF3nXuyhz/VjRlRzvlBRu4= +github.com/cilium/ebpf v0.9.1/go.mod h1:+OhNOIXx/Fnu1IE8bJz2dzOA+VSfyTfdNUVdlQnxUFY= +github.com/containerd/cgroups/v3 v3.0.1 h1:4hfGvu8rfGIwVIDd+nLzn/B9ZXx4BcCjzt5ToenJRaE= +github.com/containerd/cgroups/v3 v3.0.1/go.mod h1:/vtwk1VXrtoa5AaZLkypuOJgA/6DyPMZHJPGQNtlHnw= +github.com/coreos/go-systemd/v22 v22.3.2 h1:D9/bQk5vlXQFZ6Kwuu6zaiXJ9oTPe68++AzAJc1DzSI= +github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= +github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/frankban/quicktest v1.14.0 h1:+cqqvzZV87b4adx/5ayVOaYZ2CrvM4ejQvUdBzPPUss= +github.com/frankban/quicktest v1.14.0/go.mod h1:NeW+ay9A/U67EYXNFA1nPE8e/tnQv/09mUdL/ijj8og= +github.com/godbus/dbus/v5 v5.0.4 h1:9349emZab16e7zQvpmsbtjc18ykshndd8y2PG3sgJbA= +github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= +github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/opencontainers/runtime-spec v1.0.2 h1:UfAcuLBJB9Coz72x1hgl8O5RVzTdNiaglX6v2DM6FI0= +github.com/opencontainers/runtime-spec v1.0.2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= +github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2DcNVpwGmV9E1BkGknEliJkfwQj0= +github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/prashantv/gostub v1.1.0 h1:BTyx3RfQjRHnUWaGF9oQos79AlQ5k8WNktv7VGvVH4g= +github.com/prashantv/gostub v1.1.0/go.mod h1:A5zLQHz7ieHGG7is6LLXLz7I8+3LZzsrV0P1IAHhP5U= +github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k= +github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/silas/dag v0.0.0-20220518035006-a7e85ada93c5 h1:G/FZtUu7a6NTWl3KUHMV9jkLAh/Rvtf03NWMHaEDl+E= github.com/silas/dag v0.0.0-20220518035006-a7e85ada93c5/go.mod h1:7RTUFBdIRC9nZ7/3RyRNH1bdqIShrDejd1YbLwgPS+I= +github.com/sirupsen/logrus v1.8.1 h1:dJKuHgqk1NNQlqoA6BTlM1Wf9DOH3NBjQyu0h9+AZZE= +github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.8.0 h1:pSgiaMZlXftHpm5L7V1+rVB+AZJydKsMxsQBIJw4PKk= +github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= +go.uber.org/automaxprocs v1.6.0 h1:O3y2/QNTOdbF+e/dpXNNW7Rx2hZ4sTIPyybbxyNqTUs= +go.uber.org/automaxprocs v1.6.0/go.mod h1:ifeIMSnPZuznNm6jmdzmU3/bfk01Fe2fotchwEFJ8r8= +go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA= +go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= +go.unistack.org/micro-proto/v3 v3.4.1 h1:UTjLSRz2YZuaHk9iSlVqqsA50JQNAEK2ZFboGqtEa9Q= +go.unistack.org/micro-proto/v3 v3.4.1/go.mod h1:okx/cnOhzuCX0ggl/vToatbCupi0O44diiiLLsZ93Zo= golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14= golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sys v0.0.0-20191026070338-33540a1f6037/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc= +golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/genproto/googleapis/rpc v0.0.0-20230526203410-71b5a4ffd15e h1:NumxXLPfHSndr3wBBdeKiVHjGVFzi9RX2HwwQke94iY= google.golang.org/genproto/googleapis/rpc v0.0.0-20230526203410-71b5a4ffd15e/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= @@ -26,8 +71,8 @@ google.golang.org/grpc v1.57.0 h1:kfzNeI/klCGD2YPMUlaGNT3pxvYfga7smW3Vth8Zsiw= google.golang.org/grpc v1.57.0/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= -google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/service.go b/service.go index 22bcb74e..7a9c3a46 100644 --- a/service.go +++ b/service.go @@ -5,6 +5,8 @@ import ( "fmt" "sync" + "github.com/KimMachineGun/automemlimit/memlimit" + _ "go.uber.org/automaxprocs" "go.unistack.org/micro/v3/broker" "go.unistack.org/micro/v3/client" "go.unistack.org/micro/v3/config" @@ -17,6 +19,18 @@ import ( "go.unistack.org/micro/v3/tracer" ) +func init() { + memlimit.SetGoMemLimitWithOpts( + memlimit.WithRatio(0.9), + memlimit.WithProvider( + memlimit.ApplyFallback( + memlimit.FromCgroup, + memlimit.FromSystem, + ), + ), + ) +} + // Service is an interface that wraps the lower level components. // Its works as container with building blocks for service. type Service interface { From 3bbb0cbc721233c393f2bae37a6892d51c7afd48 Mon Sep 17 00:00:00 2001 From: Evstigneev Denis Date: Sat, 12 Oct 2024 12:37:43 +0300 Subject: [PATCH 46/59] update slog/logger (#351) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Изменено (методы logger без форматирования): - Добавлена подготовка и выравнивание аттрибутов для logger - Выравнивание за счет добавления !BADKEY до процессинга log/slog - Добавлено переиспользование метода Log - Удалены методы [Logf, Infof, Debugf, Errorf, Warnf, Fatalf, Tracef] - Обновлены юниттесты - Удален wrapper в пакете logger - Изменен интерфейс logger - Отрефакторены вызовы logger'a в micro Co-authored-by: Vasiliy Tolstov Reviewed-on: https://git.unistack.org/unistack-org/micro/pulls/351 Co-authored-by: Evstigneev Denis Co-committed-by: Evstigneev Denis --- broker/memory/memory.go | 6 +- config/config.go | 13 +- flow/default.go | 28 +- fsm/fsm_test.go | 2 +- logger/logger.go | 135 +--------- logger/noop.go | 40 +-- logger/options.go | 18 +- logger/slog/slog.go | 439 ++++++-------------------------- logger/slog/slog_test.go | 68 ++++- logger/wrapper/wrapper.go | 399 ----------------------------- network/tunnel/broker/broker.go | 16 +- register/memory/memory.go | 15 +- server/noop.go | 2 +- service.go | 4 +- tracer/memory/memory_test.go | 4 +- util/id/id.go | 2 +- 16 files changed, 209 insertions(+), 982 deletions(-) delete mode 100644 logger/wrapper/wrapper.go diff --git a/broker/memory/memory.go b/broker/memory/memory.go index 9fa1551c..de8f1c73 100644 --- a/broker/memory/memory.go +++ b/broker/memory/memory.go @@ -206,7 +206,7 @@ func (m *memoryBroker) publish(ctx context.Context, msgs []*broker.Message, opts } } else if sub.opts.AutoAck { if err = ms.Ack(); err != nil { - m.opts.Logger.Errorf(m.opts.Context, "ack failed: %v", err) + m.opts.Logger.Error(m.opts.Context, "broker ack error", err) } } // single processing @@ -217,11 +217,11 @@ func (m *memoryBroker) publish(ctx context.Context, msgs []*broker.Message, opts if eh != nil { _ = eh(p) } else if m.opts.Logger.V(logger.ErrorLevel) { - m.opts.Logger.Error(m.opts.Context, err.Error()) + m.opts.Logger.Error(m.opts.Context, "broker handler error", err) } } else if sub.opts.AutoAck { if err = p.Ack(); err != nil { - m.opts.Logger.Errorf(m.opts.Context, "ack failed: %v", err) + m.opts.Logger.Error(m.opts.Context, "broker ack error", err) } } } diff --git a/config/config.go b/config/config.go index aa8ff16f..0ca629e4 100644 --- a/config/config.go +++ b/config/config.go @@ -4,6 +4,7 @@ package config import ( "context" "errors" + "fmt" "reflect" "time" ) @@ -138,7 +139,7 @@ var ( return nil } if err := fn(ctx, c); err != nil { - c.Options().Logger.Errorf(ctx, "%s BeforeLoad err: %v", c.String(), err) + c.Options().Logger.Error(ctx, fmt.Sprintf("%s BeforeLoad error", c.String()), err) if !c.Options().AllowFail { return err } @@ -153,7 +154,7 @@ var ( return nil } if err := fn(ctx, c); err != nil { - c.Options().Logger.Errorf(ctx, "%s AfterLoad err: %v", c.String(), err) + c.Options().Logger.Error(ctx, fmt.Sprintf("%s AfterLoad error", c.String()), err) if !c.Options().AllowFail { return err } @@ -168,7 +169,7 @@ var ( return nil } if err := fn(ctx, c); err != nil { - c.Options().Logger.Errorf(ctx, "%s BeforeSave err: %v", c.String(), err) + c.Options().Logger.Error(ctx, fmt.Sprintf("%s BeforeSave error", c.String()), err) if !c.Options().AllowFail { return err } @@ -183,7 +184,7 @@ var ( return nil } if err := fn(ctx, c); err != nil { - c.Options().Logger.Errorf(ctx, "%s AfterSave err: %v", c.String(), err) + c.Options().Logger.Error(ctx, fmt.Sprintf("%s AfterSave error", c.String()), err) if !c.Options().AllowFail { return err } @@ -198,7 +199,7 @@ var ( return nil } if err := fn(ctx, c); err != nil { - c.Options().Logger.Errorf(ctx, "%s BeforeInit err: %v", c.String(), err) + c.Options().Logger.Error(ctx, fmt.Sprintf("%s BeforeInit error", c.String()), err) if !c.Options().AllowFail { return err } @@ -213,7 +214,7 @@ var ( return nil } if err := fn(ctx, c); err != nil { - c.Options().Logger.Errorf(ctx, "%s AfterInit err: %v", c.String(), err) + c.Options().Logger.Error(ctx, fmt.Sprintf("%s AfterInit error", c.String(), err), err) if !c.Options().AllowFail { return err } diff --git a/flow/default.go b/flow/default.go index af0e018a..e07303f2 100644 --- a/flow/default.go +++ b/flow/default.go @@ -188,7 +188,7 @@ func (w *microWorkflow) Execute(ctx context.Context, req *Message, opts ...Execu steps, err := w.getSteps(options.Start, options.Reverse) if err != nil { if werr := workflowStore.Write(w.opts.Context, "status", &codec.Frame{Data: []byte(StatusPending.String())}); werr != nil { - w.opts.Logger.Errorf(w.opts.Context, "store error: %v", werr) + w.opts.Logger.Error(w.opts.Context, "store write error", werr) } return "", err } @@ -212,7 +212,7 @@ func (w *microWorkflow) Execute(ctx context.Context, req *Message, opts ...Execu done := make(chan struct{}) if werr := workflowStore.Write(w.opts.Context, "status", &codec.Frame{Data: []byte(StatusRunning.String())}); werr != nil { - w.opts.Logger.Errorf(w.opts.Context, "store error: %v", werr) + w.opts.Logger.Error(w.opts.Context, "store write error", werr) return eid, werr } for idx := range steps { @@ -237,7 +237,7 @@ func (w *microWorkflow) Execute(ctx context.Context, req *Message, opts ...Execu return } if w.opts.Logger.V(logger.TraceLevel) { - w.opts.Logger.Tracef(nctx, "will be executed %v", steps[idx][nidx]) + w.opts.Logger.Trace(nctx, fmt.Sprintf("will be executed %v", steps[idx][nidx])) } cstep := steps[idx][nidx] // nolint: nestif @@ -257,21 +257,21 @@ func (w *microWorkflow) Execute(ctx context.Context, req *Message, opts ...Execu if serr != nil { step.SetStatus(StatusFailure) if werr := stepStore.Write(ctx, step.ID()+w.opts.Store.Options().Separator+"rsp", serr); werr != nil && w.opts.Logger.V(logger.ErrorLevel) { - w.opts.Logger.Errorf(ctx, "store write error: %v", werr) + w.opts.Logger.Error(ctx, "store write error", werr) } if werr := stepStore.Write(ctx, step.ID()+w.opts.Store.Options().Separator+"status", &codec.Frame{Data: []byte(StatusFailure.String())}); werr != nil && w.opts.Logger.V(logger.ErrorLevel) { - w.opts.Logger.Errorf(ctx, "store write error: %v", werr) + w.opts.Logger.Error(ctx, "store write error", werr) } cherr <- serr return } if werr := stepStore.Write(ctx, step.ID()+w.opts.Store.Options().Separator+"rsp", rsp); werr != nil { - w.opts.Logger.Errorf(ctx, "store write error: %v", werr) + w.opts.Logger.Error(ctx, "store write error", werr) cherr <- werr return } if werr := stepStore.Write(ctx, step.ID()+w.opts.Store.Options().Separator+"status", &codec.Frame{Data: []byte(StatusSuccess.String())}); werr != nil { - w.opts.Logger.Errorf(ctx, "store write error: %v", werr) + w.opts.Logger.Error(ctx, "store write error", werr) cherr <- werr return } @@ -290,16 +290,16 @@ func (w *microWorkflow) Execute(ctx context.Context, req *Message, opts ...Execu if serr != nil { cstep.SetStatus(StatusFailure) if werr := stepStore.Write(ctx, cstep.ID()+w.opts.Store.Options().Separator+"rsp", serr); werr != nil && w.opts.Logger.V(logger.ErrorLevel) { - w.opts.Logger.Errorf(ctx, "store write error: %v", werr) + w.opts.Logger.Error(ctx, "store write error", werr) } if werr := stepStore.Write(ctx, cstep.ID()+w.opts.Store.Options().Separator+"status", &codec.Frame{Data: []byte(StatusFailure.String())}); werr != nil && w.opts.Logger.V(logger.ErrorLevel) { - w.opts.Logger.Errorf(ctx, "store write error: %v", werr) + w.opts.Logger.Error(ctx, "store write error", werr) } cherr <- serr return } if werr := stepStore.Write(ctx, cstep.ID()+w.opts.Store.Options().Separator+"rsp", rsp); werr != nil { - w.opts.Logger.Errorf(ctx, "store write error: %v", werr) + w.opts.Logger.Error(ctx, "store write error", werr) cherr <- werr return } @@ -317,7 +317,7 @@ func (w *microWorkflow) Execute(ctx context.Context, req *Message, opts ...Execu return eid, nil } - logger.Tracef(ctx, "wait for finish or error") + logger.DefaultLogger.Trace(ctx, "wait for finish or error") select { case <-nctx.Done(): err = nctx.Err() @@ -333,15 +333,15 @@ func (w *microWorkflow) Execute(ctx context.Context, req *Message, opts ...Execu switch { case nctx.Err() != nil: if werr := workflowStore.Write(w.opts.Context, "status", &codec.Frame{Data: []byte(StatusAborted.String())}); werr != nil { - w.opts.Logger.Errorf(w.opts.Context, "store error: %v", werr) + w.opts.Logger.Error(w.opts.Context, "store write error", werr) } case err == nil: if werr := workflowStore.Write(w.opts.Context, "status", &codec.Frame{Data: []byte(StatusSuccess.String())}); werr != nil { - w.opts.Logger.Errorf(w.opts.Context, "store error: %v", werr) + w.opts.Logger.Error(w.opts.Context, "store write error", werr) } case err != nil: if werr := workflowStore.Write(w.opts.Context, "status", &codec.Frame{Data: []byte(StatusFailure.String())}); werr != nil { - w.opts.Logger.Errorf(w.opts.Context, "store error: %v", werr) + w.opts.Logger.Error(w.opts.Context, "store write error", werr) } } diff --git a/fsm/fsm_test.go b/fsm/fsm_test.go index ad7d706e..d39920c8 100644 --- a/fsm/fsm_test.go +++ b/fsm/fsm_test.go @@ -17,7 +17,7 @@ func TestFSMStart(t *testing.T) { wrapper := func(next StateFunc) StateFunc { return func(sctx context.Context, s State, opts ...StateOption) (State, error) { - sctx = logger.NewContext(sctx, logger.Fields("state", s.Name())) + sctx = logger.NewContext(sctx, logger.DefaultLogger.Fields("state", s.Name())) return next(sctx, s, opts...) } } diff --git a/logger/logger.go b/logger/logger.go index 175311bc..33622533 100644 --- a/logger/logger.go +++ b/logger/logger.go @@ -14,8 +14,6 @@ var ( DefaultLogger Logger = NewLogger() // DefaultLevel used by logger DefaultLevel = InfoLevel - // DefaultCallerSkipCount used by logger - DefaultCallerSkipCount = 2 ) // Logger is a generic logging interface @@ -33,33 +31,19 @@ type Logger interface { // Fields set fields to always be logged with keyval pairs Fields(fields ...interface{}) Logger // Info level message - Info(ctx context.Context, args ...interface{}) + Info(ctx context.Context, msg string, args ...interface{}) // Trace level message - Trace(ctx context.Context, args ...interface{}) + Trace(ctx context.Context, msg string, args ...interface{}) // Debug level message - Debug(ctx context.Context, args ...interface{}) + Debug(ctx context.Context, msg string, args ...interface{}) // Warn level message - Warn(ctx context.Context, args ...interface{}) + Warn(ctx context.Context, msg string, args ...interface{}) // Error level message - Error(ctx context.Context, args ...interface{}) + Error(ctx context.Context, msg string, args ...interface{}) // Fatal level message - Fatal(ctx context.Context, args ...interface{}) - // Infof level message - Infof(ctx context.Context, msg string, args ...interface{}) - // Tracef level message - Tracef(ctx context.Context, msg string, args ...interface{}) - // Debug level message - Debugf(ctx context.Context, msg string, args ...interface{}) - // Warn level message - Warnf(ctx context.Context, msg string, args ...interface{}) - // Error level message - Errorf(ctx context.Context, msg string, args ...interface{}) - // Fatal level message - Fatalf(ctx context.Context, msg string, args ...interface{}) + Fatal(ctx context.Context, msg string, args ...interface{}) // Log logs message with needed level - Log(ctx context.Context, level Level, args ...interface{}) - // Logf logs message with needed level - Logf(ctx context.Context, level Level, msg string, args ...interface{}) + Log(ctx context.Context, level Level, msg string, args ...interface{}) // Name returns broker instance name Name() string // String returns the type of logger @@ -68,108 +52,3 @@ type Logger interface { // Field contains keyval pair type Field interface{} - -// Info writes msg to default logger on info level -// -// Deprecated: Dont use logger methods directly, use instance of logger to avoid additional allocations -func Info(ctx context.Context, args ...interface{}) { - DefaultLogger.Clone(WithCallerSkipCount(DefaultCallerSkipCount+1)).Info(ctx, args...) -} - -// Error writes msg to default logger on error level -// -// Deprecated: Dont use logger methods directly, use instance of logger to avoid additional allocations -func Error(ctx context.Context, args ...interface{}) { - DefaultLogger.Clone(WithCallerSkipCount(DefaultCallerSkipCount+1)).Error(ctx, args...) -} - -// Debug writes msg to default logger on debug level -// -// Deprecated: Dont use logger methods directly, use instance of logger to avoid additional allocations -func Debug(ctx context.Context, args ...interface{}) { - DefaultLogger.Clone(WithCallerSkipCount(DefaultCallerSkipCount+1)).Debug(ctx, args...) -} - -// Warn writes msg to default logger on warn level -// -// Deprecated: Dont use logger methods directly, use instance of logger to avoid additional allocations -func Warn(ctx context.Context, args ...interface{}) { - DefaultLogger.Clone(WithCallerSkipCount(DefaultCallerSkipCount+1)).Warn(ctx, args...) -} - -// Trace writes msg to default logger on trace level -// -// Deprecated: Dont use logger methods directly, use instance of logger to avoid additional allocations -func Trace(ctx context.Context, args ...interface{}) { - DefaultLogger.Clone(WithCallerSkipCount(DefaultCallerSkipCount+1)).Trace(ctx, args...) -} - -// Fatal writes msg to default logger on fatal level -// -// Deprecated: Dont use logger methods directly, use instance of logger to avoid additional allocations -func Fatal(ctx context.Context, args ...interface{}) { - DefaultLogger.Clone(WithCallerSkipCount(DefaultCallerSkipCount+1)).Fatal(ctx, args...) -} - -// Infof writes formatted msg to default logger on info level -// -// Deprecated: Dont use logger methods directly, use instance of logger to avoid additional allocations -func Infof(ctx context.Context, msg string, args ...interface{}) { - DefaultLogger.Clone(WithCallerSkipCount(DefaultCallerSkipCount+1)).Infof(ctx, msg, args...) -} - -// Errorf writes formatted msg to default logger on error level -// -// Deprecated: Dont use logger methods directly, use instance of logger to avoid additional allocations -func Errorf(ctx context.Context, msg string, args ...interface{}) { - DefaultLogger.Clone(WithCallerSkipCount(DefaultCallerSkipCount+1)).Errorf(ctx, msg, args...) -} - -// Debugf writes formatted msg to default logger on debug level -// -// Deprecated: Dont use logger methods directly, use instance of logger to avoid additional allocations -func Debugf(ctx context.Context, msg string, args ...interface{}) { - DefaultLogger.Clone(WithCallerSkipCount(DefaultCallerSkipCount+1)).Debugf(ctx, msg, args...) -} - -// Warnf writes formatted msg to default logger on warn level -// -// Deprecated: Dont use logger methods directly, use instance of logger to avoid additional allocations -func Warnf(ctx context.Context, msg string, args ...interface{}) { - DefaultLogger.Clone(WithCallerSkipCount(DefaultCallerSkipCount+1)).Warnf(ctx, msg, args...) -} - -// Tracef writes formatted msg to default logger on trace level -// -// Deprecated: Dont use logger methods directly, use instance of logger to avoid additional allocations -func Tracef(ctx context.Context, msg string, args ...interface{}) { - DefaultLogger.Clone(WithCallerSkipCount(DefaultCallerSkipCount+1)).Tracef(ctx, msg, args...) -} - -// Fatalf writes formatted msg to default logger on fatal level -// -// Deprecated: Dont use logger methods directly, use instance of logger to avoid additional allocations -func Fatalf(ctx context.Context, msg string, args ...interface{}) { - DefaultLogger.Clone(WithCallerSkipCount(DefaultCallerSkipCount+1)).Fatalf(ctx, msg, args...) -} - -// V returns true if passed level enabled in default logger -// -// Deprecated: Dont use logger methods directly, use instance of logger to avoid additional allocations -func V(level Level) bool { - return DefaultLogger.V(level) -} - -// Init initialize logger -// -// Deprecated: Dont use logger methods directly, use instance of logger to avoid additional allocations -func Init(opts ...Option) error { - return DefaultLogger.Init(opts...) -} - -// Fields create logger with specific fields -// -// Deprecated: Dont use logger methods directly, use instance of logger to avoid additional allocations -func Fields(fields ...interface{}) Logger { - return DefaultLogger.Fields(fields...) -} diff --git a/logger/noop.go b/logger/noop.go index 17f3608b..af9dfeae 100644 --- a/logger/noop.go +++ b/logger/noop.go @@ -4,12 +4,17 @@ import ( "context" ) +const ( + defaultCallerSkipCount = 2 +) + type noopLogger struct { opts Options } func NewLogger(opts ...Option) Logger { options := NewOptions(opts...) + options.CallerSkipCount = defaultCallerSkipCount return &noopLogger{opts: options} } @@ -51,44 +56,23 @@ func (l *noopLogger) String() string { return "noop" } -func (l *noopLogger) Log(ctx context.Context, lvl Level, attrs ...interface{}) { +func (l *noopLogger) Log(ctx context.Context, lvl Level, msg string, attrs ...interface{}) { } -func (l *noopLogger) Info(ctx context.Context, attrs ...interface{}) { +func (l *noopLogger) Info(ctx context.Context, msg string, attrs ...interface{}) { } -func (l *noopLogger) Debug(ctx context.Context, attrs ...interface{}) { +func (l *noopLogger) Debug(ctx context.Context, msg string, attrs ...interface{}) { } -func (l *noopLogger) Error(ctx context.Context, attrs ...interface{}) { +func (l *noopLogger) Error(ctx context.Context, msg string, attrs ...interface{}) { } -func (l *noopLogger) Trace(ctx context.Context, attrs ...interface{}) { +func (l *noopLogger) Trace(ctx context.Context, msg string, attrs ...interface{}) { } -func (l *noopLogger) Warn(ctx context.Context, attrs ...interface{}) { +func (l *noopLogger) Warn(ctx context.Context, msg string, attrs ...interface{}) { } -func (l *noopLogger) Fatal(ctx context.Context, attrs ...interface{}) { -} - -func (l *noopLogger) Logf(ctx context.Context, lvl Level, msg string, attrs ...interface{}) { -} - -func (l *noopLogger) Infof(ctx context.Context, msg string, attrs ...interface{}) { -} - -func (l *noopLogger) Debugf(ctx context.Context, msg string, attrs ...interface{}) { -} - -func (l *noopLogger) Errorf(ctx context.Context, msg string, attrs ...interface{}) { -} - -func (l *noopLogger) Tracef(ctx context.Context, msg string, attrs ...interface{}) { -} - -func (l *noopLogger) Warnf(ctx context.Context, msg string, attrs ...interface{}) { -} - -func (l *noopLogger) Fatalf(ctx context.Context, msg string, attrs ...interface{}) { +func (l *noopLogger) Fatal(ctx context.Context, msg string, attrs ...interface{}) { } diff --git a/logger/options.go b/logger/options.go index e1113daf..ede9a23b 100644 --- a/logger/options.go +++ b/logger/options.go @@ -23,7 +23,7 @@ type Options struct { Name string // Fields holds additional metadata Fields []interface{} - // CallerSkipCount number of frmaes to skip + // callerSkipCount number of frmaes to skip CallerSkipCount int // ContextAttrFuncs contains funcs that executed before log func on context ContextAttrFuncs []ContextAttrFunc @@ -57,7 +57,6 @@ func NewOptions(opts ...Option) Options { Level: DefaultLevel, Fields: make([]interface{}, 0, 6), Out: os.Stderr, - CallerSkipCount: DefaultCallerSkipCount, Context: context.Background(), ContextAttrFuncs: DefaultContextAttrFuncs, AddSource: true, @@ -102,27 +101,20 @@ func WithOutput(out io.Writer) Option { } } -// WitAddStacktrace controls writing stacktrace on error +// WithAddStacktrace controls writing stacktrace on error func WithAddStacktrace(v bool) Option { return func(o *Options) { o.AddStacktrace = v } } -// WitAddSource controls writing source file and pos in log +// WithAddSource controls writing source file and pos in log func WithAddSource(v bool) Option { return func(o *Options) { o.AddSource = v } } -// WithCallerSkipCount set frame count to skip -func WithCallerSkipCount(c int) Option { - return func(o *Options) { - o.CallerSkipCount = c - } -} - // WithContext set context func WithContext(ctx context.Context) Option { return func(o *Options) { @@ -198,6 +190,8 @@ func WithMicroKeys() Option { // WithAddCallerSkipCount add skip count for copy logger func WithAddCallerSkipCount(n int) Option { return func(o *Options) { - o.CallerSkipCount += n + if n > 0 { + o.CallerSkipCount += n + } } } diff --git a/logger/slog/slog.go b/logger/slog/slog.go index 76a7e1bb..76279083 100644 --- a/logger/slog/slog.go +++ b/logger/slog/slog.go @@ -2,7 +2,6 @@ package slog import ( "context" - "fmt" "log/slog" "os" "regexp" @@ -15,6 +14,12 @@ import ( "go.unistack.org/micro/v3/tracer" ) +const ( + badKey = "!BADKEY" + // defaultCallerSkipCount used by logger + defaultCallerSkipCount = 3 +) + var reTrace = regexp.MustCompile(`.*/slog/logger\.go.*\n`) var ( @@ -150,386 +155,33 @@ func (s *slogLogger) Init(opts ...logger.Option) error { return nil } -func (s *slogLogger) Log(ctx context.Context, lvl logger.Level, attrs ...interface{}) { - s.opts.Meter.Counter(semconv.LoggerMessageTotal, "level", lvl.String()).Inc() - if !s.V(lvl) { - return - } - var pcs [1]uintptr - runtime.Callers(s.opts.CallerSkipCount, pcs[:]) // skip [Callers, Infof] - r := slog.NewRecord(s.opts.TimeFunc(), loggerToSlogLevel(lvl), fmt.Sprintf("%s", attrs[0]), pcs[0]) - for _, fn := range s.opts.ContextAttrFuncs { - attrs = append(attrs, fn(ctx)...) - } - - for idx, attr := range attrs { - if ve, ok := attr.(error); ok && ve != nil { - attrs[idx] = slog.String(s.opts.ErrorKey, ve.Error()) - break - } - } - if s.opts.AddStacktrace && lvl == logger.ErrorLevel { - stackInfo := make([]byte, 1024*1024) - if stackSize := runtime.Stack(stackInfo, false); stackSize > 0 { - traceLines := reTrace.Split(string(stackInfo[:stackSize]), -1) - if len(traceLines) != 0 { - attrs = append(attrs, slog.String(s.opts.StacktraceKey, traceLines[len(traceLines)-1])) - } - } - } - r.Add(attrs[1:]...) - r.Attrs(func(a slog.Attr) bool { - if a.Key == s.opts.ErrorKey { - if span, ok := tracer.SpanFromContext(ctx); ok { - span.SetStatus(tracer.SpanStatusError, a.Value.String()) - return false - } - } - return true - }) - _ = s.handler.Handle(ctx, r) +func (s *slogLogger) Log(ctx context.Context, lvl logger.Level, msg string, attrs ...interface{}) { + s.printLog(ctx, lvl, msg, attrs...) } -func (s *slogLogger) Logf(ctx context.Context, lvl logger.Level, msg string, attrs ...interface{}) { - s.opts.Meter.Counter(semconv.LoggerMessageTotal, "level", lvl.String()).Inc() - if !s.V(lvl) { - return - } - var pcs [1]uintptr - runtime.Callers(s.opts.CallerSkipCount, pcs[:]) // skip [Callers, Infof] - r := slog.NewRecord(s.opts.TimeFunc(), loggerToSlogLevel(lvl), msg, pcs[0]) - for _, fn := range s.opts.ContextAttrFuncs { - attrs = append(attrs, fn(ctx)...) - } - - for idx, attr := range attrs { - if ve, ok := attr.(error); ok && ve != nil { - attrs[idx] = slog.String(s.opts.ErrorKey, ve.Error()) - break - } - } - if s.opts.AddStacktrace && lvl == logger.ErrorLevel { - stackInfo := make([]byte, 1024*1024) - if stackSize := runtime.Stack(stackInfo, false); stackSize > 0 { - traceLines := reTrace.Split(string(stackInfo[:stackSize]), -1) - if len(traceLines) != 0 { - attrs = append(attrs, (slog.String(s.opts.StacktraceKey, traceLines[len(traceLines)-1]))) - } - } - } - r.Add(attrs[1:]...) - r.Attrs(func(a slog.Attr) bool { - if a.Key == s.opts.ErrorKey { - if span, ok := tracer.SpanFromContext(ctx); ok { - span.SetStatus(tracer.SpanStatusError, a.Value.String()) - return false - } - } - return true - }) - _ = s.handler.Handle(ctx, r) +func (s *slogLogger) Info(ctx context.Context, msg string, attrs ...interface{}) { + s.printLog(ctx, logger.InfoLevel, msg, attrs...) } -func (s *slogLogger) Info(ctx context.Context, attrs ...interface{}) { - s.opts.Meter.Counter(semconv.LoggerMessageTotal, "level", logger.InfoLevel.String()).Inc() - if !s.V(logger.InfoLevel) { - return - } - var pcs [1]uintptr - runtime.Callers(s.opts.CallerSkipCount, pcs[:]) // skip [Callers, Infof] - r := slog.NewRecord(s.opts.TimeFunc(), slog.LevelInfo, fmt.Sprintf("%s", attrs[0]), pcs[0]) - for _, fn := range s.opts.ContextAttrFuncs { - attrs = append(attrs, fn(ctx)...) - } - - for idx, attr := range attrs { - if ve, ok := attr.(error); ok && ve != nil { - attrs[idx] = slog.String(s.opts.ErrorKey, ve.Error()) - break - } - } - r.Add(attrs[1:]...) - _ = s.handler.Handle(ctx, r) +func (s *slogLogger) Debug(ctx context.Context, msg string, attrs ...interface{}) { + s.printLog(ctx, logger.DebugLevel, msg, attrs...) } -func (s *slogLogger) Infof(ctx context.Context, msg string, attrs ...interface{}) { - s.opts.Meter.Counter(semconv.LoggerMessageTotal, "level", logger.InfoLevel.String()).Inc() - if !s.V(logger.InfoLevel) { - return - } - var pcs [1]uintptr - runtime.Callers(s.opts.CallerSkipCount, pcs[:]) // skip [Callers, Infof] - r := slog.NewRecord(s.opts.TimeFunc(), slog.LevelInfo, msg, pcs[0]) - for _, fn := range s.opts.ContextAttrFuncs { - attrs = append(attrs, fn(ctx)...) - } - - for idx, attr := range attrs { - if ve, ok := attr.(error); ok && ve != nil { - attrs[idx] = slog.String(s.opts.ErrorKey, ve.Error()) - break - } - } - r.Add(attrs...) - _ = s.handler.Handle(ctx, r) +func (s *slogLogger) Trace(ctx context.Context, msg string, attrs ...interface{}) { + s.printLog(ctx, logger.TraceLevel, msg, attrs...) } -func (s *slogLogger) Debug(ctx context.Context, attrs ...interface{}) { - s.opts.Meter.Counter(semconv.LoggerMessageTotal, "level", logger.DebugLevel.String()).Inc() - if !s.V(logger.DebugLevel) { - return - } - var pcs [1]uintptr - runtime.Callers(s.opts.CallerSkipCount, pcs[:]) // skip [Callers, Infof] - r := slog.NewRecord(s.opts.TimeFunc(), slog.LevelDebug, fmt.Sprintf("%s", attrs[0]), pcs[0]) - for _, fn := range s.opts.ContextAttrFuncs { - attrs = append(attrs, fn(ctx)...) - } - - for idx, attr := range attrs { - if ve, ok := attr.(error); ok && ve != nil { - attrs[idx] = slog.String(s.opts.ErrorKey, ve.Error()) - break - } - } - r.Add(attrs[1:]...) - _ = s.handler.Handle(ctx, r) +func (s *slogLogger) Error(ctx context.Context, msg string, attrs ...interface{}) { + s.printLog(ctx, logger.ErrorLevel, msg, attrs...) } -func (s *slogLogger) Debugf(ctx context.Context, msg string, attrs ...interface{}) { - s.opts.Meter.Counter(semconv.LoggerMessageTotal, "level", logger.DebugLevel.String()).Inc() - if !s.V(logger.DebugLevel) { - return - } - var pcs [1]uintptr - runtime.Callers(s.opts.CallerSkipCount, pcs[:]) // skip [Callers, Infof] - r := slog.NewRecord(s.opts.TimeFunc(), slog.LevelDebug, msg, pcs[0]) - for _, fn := range s.opts.ContextAttrFuncs { - attrs = append(attrs, fn(ctx)...) - } - - for idx, attr := range attrs { - if ve, ok := attr.(error); ok && ve != nil { - attrs[idx] = slog.String(s.opts.ErrorKey, ve.Error()) - break - } - } - r.Add(attrs...) - _ = s.handler.Handle(ctx, r) -} - -func (s *slogLogger) Trace(ctx context.Context, attrs ...interface{}) { - s.opts.Meter.Counter(semconv.LoggerMessageTotal, "level", logger.TraceLevel.String()).Inc() - if !s.V(logger.TraceLevel) { - return - } - var pcs [1]uintptr - runtime.Callers(s.opts.CallerSkipCount, pcs[:]) // skip [Callers, Infof] - r := slog.NewRecord(s.opts.TimeFunc(), slog.LevelDebug-1, fmt.Sprintf("%s", attrs[0]), pcs[0]) - for _, fn := range s.opts.ContextAttrFuncs { - attrs = append(attrs, fn(ctx)...) - } - - for idx, attr := range attrs { - if ve, ok := attr.(error); ok && ve != nil { - attrs[idx] = slog.String(s.opts.ErrorKey, ve.Error()) - break - } - } - r.Add(attrs[1:]...) - _ = s.handler.Handle(ctx, r) -} - -func (s *slogLogger) Tracef(ctx context.Context, msg string, attrs ...interface{}) { - s.opts.Meter.Counter(semconv.LoggerMessageTotal, "level", logger.TraceLevel.String()).Inc() - if !s.V(logger.TraceLevel) { - return - } - var pcs [1]uintptr - runtime.Callers(s.opts.CallerSkipCount, pcs[:]) // skip [Callers, Infof] - r := slog.NewRecord(s.opts.TimeFunc(), slog.LevelDebug-1, msg, pcs[0]) - for _, fn := range s.opts.ContextAttrFuncs { - attrs = append(attrs, fn(ctx)...) - } - - for idx, attr := range attrs { - if ve, ok := attr.(error); ok && ve != nil { - attrs[idx] = slog.String(s.opts.ErrorKey, ve.Error()) - break - } - } - r.Add(attrs[1:]...) - _ = s.handler.Handle(ctx, r) -} - -func (s *slogLogger) Error(ctx context.Context, attrs ...interface{}) { - s.opts.Meter.Counter(semconv.LoggerMessageTotal, "level", logger.ErrorLevel.String()).Inc() - if !s.V(logger.ErrorLevel) { - return - } - var pcs [1]uintptr - runtime.Callers(s.opts.CallerSkipCount, pcs[:]) // skip [Callers, Infof] - r := slog.NewRecord(s.opts.TimeFunc(), slog.LevelError, fmt.Sprintf("%s", attrs[0]), pcs[0]) - for _, fn := range s.opts.ContextAttrFuncs { - attrs = append(attrs, fn(ctx)...) - } - - for idx, attr := range attrs { - if ve, ok := attr.(error); ok && ve != nil { - attrs[idx] = slog.String(s.opts.ErrorKey, ve.Error()) - break - } - } - if s.opts.AddStacktrace { - stackInfo := make([]byte, 1024*1024) - if stackSize := runtime.Stack(stackInfo, false); stackSize > 0 { - traceLines := reTrace.Split(string(stackInfo[:stackSize]), -1) - if len(traceLines) != 0 { - attrs = append(attrs, slog.String("stacktrace", traceLines[len(traceLines)-1])) - } - } - } - r.Add(attrs[1:]...) - r.Attrs(func(a slog.Attr) bool { - if a.Key == s.opts.ErrorKey { - if span, ok := tracer.SpanFromContext(ctx); ok { - span.SetStatus(tracer.SpanStatusError, a.Value.String()) - return false - } - } - return true - }) - _ = s.handler.Handle(ctx, r) -} - -func (s *slogLogger) Errorf(ctx context.Context, msg string, attrs ...interface{}) { - s.opts.Meter.Counter(semconv.LoggerMessageTotal, "level", logger.ErrorLevel.String()).Inc() - if !s.V(logger.ErrorLevel) { - return - } - var pcs [1]uintptr - runtime.Callers(s.opts.CallerSkipCount, pcs[:]) // skip [Callers, Infof] - r := slog.NewRecord(s.opts.TimeFunc(), slog.LevelError, msg, pcs[0]) - for _, fn := range s.opts.ContextAttrFuncs { - attrs = append(attrs, fn(ctx)...) - } - - for idx, attr := range attrs { - if ve, ok := attr.(error); ok && ve != nil { - attrs[idx] = slog.String(s.opts.ErrorKey, ve.Error()) - break - } - } - if s.opts.AddStacktrace { - stackInfo := make([]byte, 1024*1024) - if stackSize := runtime.Stack(stackInfo, false); stackSize > 0 { - traceLines := reTrace.Split(string(stackInfo[:stackSize]), -1) - if len(traceLines) != 0 { - attrs = append(attrs, slog.String("stacktrace", traceLines[len(traceLines)-1])) - } - } - } - r.Add(attrs...) - r.Attrs(func(a slog.Attr) bool { - if a.Key == s.opts.ErrorKey { - if span, ok := tracer.SpanFromContext(ctx); ok { - span.SetStatus(tracer.SpanStatusError, a.Value.String()) - return false - } - } - return true - }) - _ = s.handler.Handle(ctx, r) -} - -func (s *slogLogger) Fatal(ctx context.Context, attrs ...interface{}) { - s.opts.Meter.Counter(semconv.LoggerMessageTotal, "level", logger.FatalLevel.String()).Inc() - if !s.V(logger.FatalLevel) { - return - } - var pcs [1]uintptr - runtime.Callers(s.opts.CallerSkipCount, pcs[:]) // skip [Callers, Infof] - r := slog.NewRecord(s.opts.TimeFunc(), slog.LevelError+1, fmt.Sprintf("%s", attrs[0]), pcs[0]) - for _, fn := range s.opts.ContextAttrFuncs { - attrs = append(attrs, fn(ctx)...) - } - - for idx, attr := range attrs { - if ve, ok := attr.(error); ok && ve != nil { - attrs[idx] = slog.String(s.opts.ErrorKey, ve.Error()) - break - } - } - r.Add(attrs[1:]...) - _ = s.handler.Handle(ctx, r) +func (s *slogLogger) Fatal(ctx context.Context, msg string, attrs ...interface{}) { + s.printLog(ctx, logger.FatalLevel, msg, attrs...) os.Exit(1) } -func (s *slogLogger) Fatalf(ctx context.Context, msg string, attrs ...interface{}) { - s.opts.Meter.Counter(semconv.LoggerMessageTotal, "level", logger.FatalLevel.String()).Inc() - if !s.V(logger.FatalLevel) { - return - } - var pcs [1]uintptr - runtime.Callers(s.opts.CallerSkipCount, pcs[:]) // skip [Callers, Infof] - r := slog.NewRecord(s.opts.TimeFunc(), slog.LevelError+1, msg, pcs[0]) - for _, fn := range s.opts.ContextAttrFuncs { - attrs = append(attrs, fn(ctx)...) - } - - for idx, attr := range attrs { - if ve, ok := attr.(error); ok && ve != nil { - attrs[idx] = slog.String(s.opts.ErrorKey, ve.Error()) - break - } - } - r.Add(attrs...) - _ = s.handler.Handle(ctx, r) - os.Exit(1) -} - -func (s *slogLogger) Warn(ctx context.Context, attrs ...interface{}) { - s.opts.Meter.Counter(semconv.LoggerMessageTotal, "level", logger.WarnLevel.String()).Inc() - if !s.V(logger.WarnLevel) { - return - } - var pcs [1]uintptr - runtime.Callers(s.opts.CallerSkipCount, pcs[:]) // skip [Callers, Infof] - r := slog.NewRecord(s.opts.TimeFunc(), slog.LevelWarn, fmt.Sprintf("%s", attrs[0]), pcs[0]) - for _, fn := range s.opts.ContextAttrFuncs { - attrs = append(attrs, fn(ctx)...) - } - - for idx, attr := range attrs { - if ve, ok := attr.(error); ok && ve != nil { - attrs[idx] = slog.String(s.opts.ErrorKey, ve.Error()) - break - } - } - r.Add(attrs[1:]...) - _ = s.handler.Handle(ctx, r) -} - -func (s *slogLogger) Warnf(ctx context.Context, msg string, attrs ...interface{}) { - s.opts.Meter.Counter(semconv.LoggerMessageTotal, "level", logger.WarnLevel.String()).Inc() - if !s.V(logger.WarnLevel) { - return - } - var pcs [1]uintptr - runtime.Callers(s.opts.CallerSkipCount, pcs[:]) // skip [Callers, Infof] - r := slog.NewRecord(s.opts.TimeFunc(), slog.LevelWarn, msg, pcs[0]) - for _, fn := range s.opts.ContextAttrFuncs { - attrs = append(attrs, fn(ctx)...) - } - - for idx, attr := range attrs { - if ve, ok := attr.(error); ok && ve != nil { - attrs[idx] = slog.String(s.opts.ErrorKey, ve.Error()) - break - } - } - r.Add(attrs[1:]...) - _ = s.handler.Handle(ctx, r) +func (s *slogLogger) Warn(ctx context.Context, msg string, attrs ...interface{}) { + s.printLog(ctx, logger.WarnLevel, msg, attrs...) } func (s *slogLogger) Name() string { @@ -540,10 +192,52 @@ func (s *slogLogger) String() string { return "slog" } +func (s *slogLogger) printLog(ctx context.Context, lvl logger.Level, msg string, attrs ...interface{}) { + if !s.V(lvl) { + return + } + + s.opts.Meter.Counter(semconv.LoggerMessageTotal, "level", lvl.String()).Inc() + + attrs = prepareAttributes(attrs) + + for _, fn := range s.opts.ContextAttrFuncs { + a := prepareAttributes(fn(ctx)) + attrs = append(attrs, a...) + } + + for _, attr := range attrs { + if ve, hasErr := attr.(error); hasErr && ve != nil { + attrs = append(attrs, slog.String(s.opts.ErrorKey, ve.Error())) + if span, ok := tracer.SpanFromContext(ctx); ok { + span.SetStatus(tracer.SpanStatusError, ve.Error()) + } + break + } + } + + if s.opts.AddStacktrace && lvl == logger.ErrorLevel { + stackInfo := make([]byte, 1024*1024) + if stackSize := runtime.Stack(stackInfo, false); stackSize > 0 { + traceLines := reTrace.Split(string(stackInfo[:stackSize]), -1) + if len(traceLines) != 0 { + attrs = append(attrs, slog.String(s.opts.StacktraceKey, traceLines[len(traceLines)-1])) + } + } + } + + var pcs [1]uintptr + runtime.Callers(s.opts.CallerSkipCount, pcs[:]) // skip [Callers, printLog, LogLvlMethod] + r := slog.NewRecord(s.opts.TimeFunc(), loggerToSlogLevel(lvl), msg, pcs[0]) + r.Add(attrs...) + _ = s.handler.Handle(ctx, r) +} + func NewLogger(opts ...logger.Option) logger.Logger { s := &slogLogger{ opts: logger.NewOptions(opts...), } + s.opts.CallerSkipCount = defaultCallerSkipCount return s } @@ -581,3 +275,12 @@ func slogToLoggerLevel(level slog.Level) logger.Level { return logger.InfoLevel } } + +func prepareAttributes(attrs []interface{}) []interface{} { + if len(attrs)%2 == 1 { + attrs = append(attrs, badKey) + attrs[len(attrs)-1], attrs[len(attrs)-2] = attrs[len(attrs)-2], attrs[len(attrs)-1] + } + + return attrs +} diff --git a/logger/slog/slog_test.go b/logger/slog/slog_test.go index 93c63c79..f8daace0 100644 --- a/logger/slog/slog_test.go +++ b/logger/slog/slog_test.go @@ -3,8 +3,12 @@ package slog import ( "bytes" "context" + "errors" "fmt" + "github.com/google/uuid" + "go.unistack.org/micro/v3/metadata" "log" + "strings" "testing" "go.unistack.org/micro/v3/logger" @@ -29,13 +33,25 @@ func TestError(t *testing.T) { func TestErrorf(t *testing.T) { ctx := context.TODO() + buf := bytes.NewBuffer(nil) l := NewLogger(logger.WithLevel(logger.ErrorLevel), logger.WithOutput(buf), logger.WithAddStacktrace(true)) - if err := l.Init(); err != nil { + if err := l.Init(logger.WithContextAttrFuncs(func(ctx context.Context) []interface{} { + return nil + })); err != nil { t.Fatal(err) } - l.Errorf(ctx, "message", fmt.Errorf("error message")) + l.Log(ctx, logger.ErrorLevel, "message", errors.New("error msg")) + if !bytes.Contains(buf.Bytes(), []byte(`"!BADKEY":"`)) { + t.Fatalf("logger BADKEY not works, buf contains: %s", buf.Bytes()) + } + + l.Log(ctx, logger.ErrorLevel, "", errors.New("error msg")) + if !bytes.Contains(buf.Bytes(), []byte(`"error":"error msg"`)) { + t.Fatalf("logger error not works, buf contains: %s", buf.Bytes()) + } + if !bytes.Contains(buf.Bytes(), []byte(`"stacktrace":"`)) { t.Fatalf("logger stacktrace not works, buf contains: %s", buf.Bytes()) } @@ -99,6 +115,11 @@ func TestFromContextWithFields(t *testing.T) { if !bytes.Contains(buf.Bytes(), []byte(`"key":"val"`)) { t.Fatalf("logger fields not works, buf contains: %s", buf.Bytes()) } + + l.Info(ctx, "test", "uncorrected number attributes") + if !bytes.Contains(buf.Bytes(), []byte(`"!BADKEY":"uncorrected number attributes"`)) { + t.Fatalf("logger fields not works, buf contains: %s", buf.Bytes()) + } } func TestClone(t *testing.T) { @@ -174,3 +195,46 @@ func TestLogger(t *testing.T) { t.Fatalf("logger warn, buf %s", buf.Bytes()) } } + +func Test_WithContextAttrFunc(t *testing.T) { + loggerContextAttrFuncs := []logger.ContextAttrFunc{ + func(ctx context.Context) []interface{} { + md, ok := metadata.FromIncomingContext(ctx) + if !ok { + return nil + } + attrs := make([]interface{}, 0, 10) + for k, v := range md { + switch k { + case "X-Request-Id", "Phone", "External-Id", "Source-Service", "X-App-Install-Id", "Client-Id", "Client-Ip": + attrs = append(attrs, strings.ToLower(k), v) + } + } + return attrs + }, + } + + logger.DefaultContextAttrFuncs = append(logger.DefaultContextAttrFuncs, loggerContextAttrFuncs...) + + ctx := context.TODO() + ctx = metadata.AppendIncomingContext(ctx, "X-Request-Id", uuid.New().String(), + "Source-Service", "Test-System") + + buf := bytes.NewBuffer(nil) + l := NewLogger(logger.WithLevel(logger.TraceLevel), logger.WithOutput(buf)) + if err := l.Init(); err != nil { + t.Fatal(err) + } + + l.Info(ctx, "test message") + if !(bytes.Contains(buf.Bytes(), []byte(`"level":"info"`)) && bytes.Contains(buf.Bytes(), []byte(`"msg":"test message"`))) { + t.Fatalf("logger info, buf %s", buf.Bytes()) + } + if !(bytes.Contains(buf.Bytes(), []byte(`"x-request-id":"`))) { + t.Fatalf("logger info, buf %s", buf.Bytes()) + } + if !(bytes.Contains(buf.Bytes(), []byte(`"source-service":"Test-System"`))) { + t.Fatalf("logger info, buf %s", buf.Bytes()) + } + +} diff --git a/logger/wrapper/wrapper.go b/logger/wrapper/wrapper.go deleted file mode 100644 index ee96f5c6..00000000 --- a/logger/wrapper/wrapper.go +++ /dev/null @@ -1,399 +0,0 @@ -// Package wrapper provides wrapper for Logger -package wrapper - -import ( - "context" - "fmt" - - "go.unistack.org/micro/v3/client" - "go.unistack.org/micro/v3/logger" - "go.unistack.org/micro/v3/server" -) - -var ( - // DefaultClientCallObserver called by wrapper in client Call - DefaultClientCallObserver = func(ctx context.Context, req client.Request, rsp interface{}, opts []client.CallOption, err error) []string { - labels := []string{"service", req.Service(), "endpoint", req.Endpoint()} - if err != nil { - labels = append(labels, "error", err.Error()) - } - return labels - } - - // DefaultClientStreamObserver called by wrapper in client Stream - DefaultClientStreamObserver = func(ctx context.Context, req client.Request, opts []client.CallOption, stream client.Stream, err error) []string { - labels := []string{"service", req.Service(), "endpoint", req.Endpoint()} - if err != nil { - labels = append(labels, "error", err.Error()) - } - return labels - } - - // DefaultClientPublishObserver called by wrapper in client Publish - DefaultClientPublishObserver = func(ctx context.Context, msg client.Message, opts []client.PublishOption, err error) []string { - labels := []string{"endpoint", msg.Topic()} - if err != nil { - labels = append(labels, "error", err.Error()) - } - return labels - } - - // DefaultServerHandlerObserver called by wrapper in server Handler - DefaultServerHandlerObserver = func(ctx context.Context, req server.Request, rsp interface{}, err error) []string { - labels := []string{"service", req.Service(), "endpoint", req.Endpoint()} - if err != nil { - labels = append(labels, "error", err.Error()) - } - return labels - } - - // DefaultServerSubscriberObserver called by wrapper in server Subscriber - DefaultServerSubscriberObserver = func(ctx context.Context, msg server.Message, err error) []string { - labels := []string{"endpoint", msg.Topic()} - if err != nil { - labels = append(labels, "error", err.Error()) - } - return labels - } - - // DefaultClientCallFuncObserver called by wrapper in client CallFunc - DefaultClientCallFuncObserver = func(ctx context.Context, addr string, req client.Request, rsp interface{}, opts client.CallOptions, err error) []string { - labels := []string{"service", req.Service(), "endpoint", req.Endpoint()} - if err != nil { - labels = append(labels, "error", err.Error()) - } - return labels - } - - // DefaultSkipEndpoints wrapper not called for this endpoints - DefaultSkipEndpoints = []string{"Meter.Metrics", "Health.Live", "Health.Ready", "Health.Version"} -) - -type lWrapper struct { - client.Client - serverHandler server.HandlerFunc - serverSubscriber server.SubscriberFunc - clientCallFunc client.CallFunc - opts Options -} - -type ( - // ClientCallObserver func signature - ClientCallObserver func(context.Context, client.Request, interface{}, []client.CallOption, error) []string - // ClientStreamObserver func signature - ClientStreamObserver func(context.Context, client.Request, []client.CallOption, client.Stream, error) []string - // ClientPublishObserver func signature - ClientPublishObserver func(context.Context, client.Message, []client.PublishOption, error) []string - // ClientCallFuncObserver func signature - ClientCallFuncObserver func(context.Context, string, client.Request, interface{}, client.CallOptions, error) []string - // ServerHandlerObserver func signature - ServerHandlerObserver func(context.Context, server.Request, interface{}, error) []string - // ServerSubscriberObserver func signature - ServerSubscriberObserver func(context.Context, server.Message, error) []string -) - -// Options struct for wrapper -type Options struct { - // Logger that used for log - Logger logger.Logger - // ServerHandlerObservers funcs - ServerHandlerObservers []ServerHandlerObserver - // ServerSubscriberObservers funcs - ServerSubscriberObservers []ServerSubscriberObserver - // ClientCallObservers funcs - ClientCallObservers []ClientCallObserver - // ClientStreamObservers funcs - ClientStreamObservers []ClientStreamObserver - // ClientPublishObservers funcs - ClientPublishObservers []ClientPublishObserver - // ClientCallFuncObservers funcs - ClientCallFuncObservers []ClientCallFuncObserver - // SkipEndpoints - SkipEndpoints []string - // Level for logger - Level logger.Level - // Enabled flag - Enabled bool -} - -// Option func signature -type Option func(*Options) - -// NewOptions creates Options from Option slice -func NewOptions(opts ...Option) Options { - options := Options{ - Logger: logger.DefaultLogger, - Level: logger.TraceLevel, - ClientCallObservers: []ClientCallObserver{DefaultClientCallObserver}, - ClientStreamObservers: []ClientStreamObserver{DefaultClientStreamObserver}, - ClientPublishObservers: []ClientPublishObserver{DefaultClientPublishObserver}, - ClientCallFuncObservers: []ClientCallFuncObserver{DefaultClientCallFuncObserver}, - ServerHandlerObservers: []ServerHandlerObserver{DefaultServerHandlerObserver}, - ServerSubscriberObservers: []ServerSubscriberObserver{DefaultServerSubscriberObserver}, - SkipEndpoints: DefaultSkipEndpoints, - } - - for _, o := range opts { - o(&options) - } - - return options -} - -// WithEnabled enable/diable flag -func WithEnabled(b bool) Option { - return func(o *Options) { - o.Enabled = b - } -} - -// WithLevel log level -func WithLevel(l logger.Level) Option { - return func(o *Options) { - o.Level = l - } -} - -// WithLogger logger -func WithLogger(l logger.Logger) Option { - return func(o *Options) { - o.Logger = l - } -} - -// WithClientCallObservers funcs -func WithClientCallObservers(ob ...ClientCallObserver) Option { - return func(o *Options) { - o.ClientCallObservers = ob - } -} - -// WithClientStreamObservers funcs -func WithClientStreamObservers(ob ...ClientStreamObserver) Option { - return func(o *Options) { - o.ClientStreamObservers = ob - } -} - -// WithClientPublishObservers funcs -func WithClientPublishObservers(ob ...ClientPublishObserver) Option { - return func(o *Options) { - o.ClientPublishObservers = ob - } -} - -// WithClientCallFuncObservers funcs -func WithClientCallFuncObservers(ob ...ClientCallFuncObserver) Option { - return func(o *Options) { - o.ClientCallFuncObservers = ob - } -} - -// WithServerHandlerObservers funcs -func WithServerHandlerObservers(ob ...ServerHandlerObserver) Option { - return func(o *Options) { - o.ServerHandlerObservers = ob - } -} - -// WithServerSubscriberObservers funcs -func WithServerSubscriberObservers(ob ...ServerSubscriberObserver) Option { - return func(o *Options) { - o.ServerSubscriberObservers = ob - } -} - -// SkipEndpoins -func SkipEndpoints(eps ...string) Option { - return func(o *Options) { - o.SkipEndpoints = append(o.SkipEndpoints, eps...) - } -} - -func (l *lWrapper) Call(ctx context.Context, req client.Request, rsp interface{}, opts ...client.CallOption) error { - err := l.Client.Call(ctx, req, rsp, opts...) - - endpoint := fmt.Sprintf("%s.%s", req.Service(), req.Endpoint()) - for _, ep := range l.opts.SkipEndpoints { - if ep == endpoint { - return err - } - } - - if !l.opts.Enabled { - return err - } - - var labels []string - for _, o := range l.opts.ClientCallObservers { - labels = append(labels, o(ctx, req, rsp, opts, err)...) - } - l.opts.Logger.Fields(labels).Log(ctx, l.opts.Level) - - return err -} - -func (l *lWrapper) Stream(ctx context.Context, req client.Request, opts ...client.CallOption) (client.Stream, error) { - stream, err := l.Client.Stream(ctx, req, opts...) - - endpoint := fmt.Sprintf("%s.%s", req.Service(), req.Endpoint()) - for _, ep := range l.opts.SkipEndpoints { - if ep == endpoint { - return stream, err - } - } - - if !l.opts.Enabled { - return stream, err - } - - var labels []string - for _, o := range l.opts.ClientStreamObservers { - labels = append(labels, o(ctx, req, opts, stream, err)...) - } - l.opts.Logger.Fields(labels).Log(ctx, l.opts.Level) - - return stream, err -} - -func (l *lWrapper) Publish(ctx context.Context, msg client.Message, opts ...client.PublishOption) error { - err := l.Client.Publish(ctx, msg, opts...) - - endpoint := msg.Topic() - for _, ep := range l.opts.SkipEndpoints { - if ep == endpoint { - return err - } - } - - if !l.opts.Enabled { - return err - } - - var labels []string - for _, o := range l.opts.ClientPublishObservers { - labels = append(labels, o(ctx, msg, opts, err)...) - } - l.opts.Logger.Fields(labels).Log(ctx, l.opts.Level) - - return err -} - -func (l *lWrapper) ServerHandler(ctx context.Context, req server.Request, rsp interface{}) error { - err := l.serverHandler(ctx, req, rsp) - - endpoint := req.Endpoint() - for _, ep := range l.opts.SkipEndpoints { - if ep == endpoint { - return err - } - } - - if !l.opts.Enabled { - return err - } - - var labels []string - for _, o := range l.opts.ServerHandlerObservers { - labels = append(labels, o(ctx, req, rsp, err)...) - } - l.opts.Logger.Fields(labels).Log(ctx, l.opts.Level) - - return err -} - -func (l *lWrapper) ServerSubscriber(ctx context.Context, msg server.Message) error { - err := l.serverSubscriber(ctx, msg) - - endpoint := msg.Topic() - for _, ep := range l.opts.SkipEndpoints { - if ep == endpoint { - return err - } - } - - if !l.opts.Enabled { - return err - } - - var labels []string - for _, o := range l.opts.ServerSubscriberObservers { - labels = append(labels, o(ctx, msg, err)...) - } - l.opts.Logger.Fields(labels).Log(ctx, l.opts.Level) - - return err -} - -// NewClientWrapper accepts an open options and returns a Client Wrapper -func NewClientWrapper(opts ...Option) client.Wrapper { - return func(c client.Client) client.Client { - options := NewOptions() - for _, o := range opts { - o(&options) - } - return &lWrapper{opts: options, Client: c} - } -} - -// NewClientCallWrapper accepts an options and returns a Call Wrapper -func NewClientCallWrapper(opts ...Option) client.CallWrapper { - return func(h client.CallFunc) client.CallFunc { - options := NewOptions() - for _, o := range opts { - o(&options) - } - - l := &lWrapper{opts: options, clientCallFunc: h} - return l.ClientCallFunc - } -} - -func (l *lWrapper) ClientCallFunc(ctx context.Context, addr string, req client.Request, rsp interface{}, opts client.CallOptions) error { - err := l.clientCallFunc(ctx, addr, req, rsp, opts) - - endpoint := fmt.Sprintf("%s.%s", req.Service(), req.Endpoint()) - for _, ep := range l.opts.SkipEndpoints { - if ep == endpoint { - return err - } - } - - if !l.opts.Enabled { - return err - } - - var labels []string - for _, o := range l.opts.ClientCallFuncObservers { - labels = append(labels, o(ctx, addr, req, rsp, opts, err)...) - } - l.opts.Logger.Fields(labels).Log(ctx, l.opts.Level) - - return err -} - -// NewServerHandlerWrapper accepts an options and returns a Handler Wrapper -func NewServerHandlerWrapper(opts ...Option) server.HandlerWrapper { - return func(h server.HandlerFunc) server.HandlerFunc { - options := NewOptions() - for _, o := range opts { - o(&options) - } - - l := &lWrapper{opts: options, serverHandler: h} - return l.ServerHandler - } -} - -// NewServerSubscriberWrapper accepts an options and returns a Subscriber Wrapper -func NewServerSubscriberWrapper(opts ...Option) server.SubscriberWrapper { - return func(h server.SubscriberFunc) server.SubscriberFunc { - options := NewOptions() - for _, o := range opts { - o(&options) - } - - l := &lWrapper{opts: options, serverSubscriber: h} - return l.ServerSubscriber - } -} diff --git a/network/tunnel/broker/broker.go b/network/tunnel/broker/broker.go index 95a2f20f..921d1853 100644 --- a/network/tunnel/broker/broker.go +++ b/network/tunnel/broker/broker.go @@ -177,12 +177,12 @@ func (t *tunBatchSubscriber) run() { // receive message m := new(transport.Message) if err := c.Recv(m); err != nil { - if logger.V(logger.ErrorLevel) { - logger.Error(t.opts.Context, err.Error()) + if logger.DefaultLogger.V(logger.ErrorLevel) { + logger.DefaultLogger.Error(t.opts.Context, err.Error(), err) } if err = c.Close(); err != nil { - if logger.V(logger.ErrorLevel) { - logger.Error(t.opts.Context, err.Error()) + if logger.DefaultLogger.V(logger.ErrorLevel) { + logger.DefaultLogger.Error(t.opts.Context, err.Error(), err) } } continue @@ -222,12 +222,12 @@ func (t *tunSubscriber) run() { // receive message m := new(transport.Message) if err := c.Recv(m); err != nil { - if logger.V(logger.ErrorLevel) { - logger.Error(t.opts.Context, err.Error()) + if logger.DefaultLogger.V(logger.ErrorLevel) { + logger.DefaultLogger.Error(t.opts.Context, err.Error(), err) } if err = c.Close(); err != nil { - if logger.V(logger.ErrorLevel) { - logger.Error(t.opts.Context, err.Error()) + if logger.DefaultLogger.V(logger.ErrorLevel) { + logger.DefaultLogger.Error(t.opts.Context, err.Error(), err) } } continue diff --git a/register/memory/memory.go b/register/memory/memory.go index b674bddd..ed241c4f 100644 --- a/register/memory/memory.go +++ b/register/memory/memory.go @@ -2,6 +2,7 @@ package register import ( "context" + "fmt" "sync" "time" @@ -64,7 +65,7 @@ func (m *memory) ttlPrune() { for id, n := range record.Nodes { if n.TTL != 0 && time.Since(n.LastSeen) > n.TTL { if m.opts.Logger.V(logger.DebugLevel) { - m.opts.Logger.Debugf(m.opts.Context, "Register TTL expired for node %s of service %s", n.ID, service) + m.opts.Logger.Debug(m.opts.Context, fmt.Sprintf("Register TTL expired for node %s of service %s", n.ID, service)) } delete(m.records[domain][service][version].Nodes, id) } @@ -151,7 +152,7 @@ func (m *memory) Register(ctx context.Context, s *register.Service, opts ...regi if _, ok := srvs[s.Name][s.Version]; !ok { srvs[s.Name][s.Version] = r if m.opts.Logger.V(logger.DebugLevel) { - m.opts.Logger.Debugf(m.opts.Context, "Register added new service: %s, version: %s", s.Name, s.Version) + m.opts.Logger.Debug(m.opts.Context, fmt.Sprintf("Register added new service: %s, version: %s", s.Name, s.Version)) } m.records[options.Domain] = srvs go m.sendEvent(®ister.Result{Action: "create", Service: s}) @@ -191,14 +192,14 @@ func (m *memory) Register(ctx context.Context, s *register.Service, opts ...regi if addedNodes { if m.opts.Logger.V(logger.DebugLevel) { - m.opts.Logger.Debugf(m.opts.Context, "Register added new node to service: %s, version: %s", s.Name, s.Version) + m.opts.Logger.Debug(m.opts.Context, fmt.Sprintf("Register added new node to service: %s, version: %s", s.Name, s.Version)) } go m.sendEvent(®ister.Result{Action: "update", Service: s}) } else { // refresh TTL and timestamp for _, n := range s.Nodes { if m.opts.Logger.V(logger.DebugLevel) { - m.opts.Logger.Debugf(m.opts.Context, "Updated registration for service: %s, version: %s", s.Name, s.Version) + m.opts.Logger.Debug(m.opts.Context, fmt.Sprintf("Updated registration for service: %s, version: %s", s.Name, s.Version)) } srvs[s.Name][s.Version].Nodes[n.ID].TTL = options.TTL srvs[s.Name][s.Version].Nodes[n.ID].LastSeen = time.Now() @@ -243,7 +244,7 @@ func (m *memory) Deregister(ctx context.Context, s *register.Service, opts ...re for _, n := range s.Nodes { if _, ok := version.Nodes[n.ID]; ok { if m.opts.Logger.V(logger.DebugLevel) { - m.opts.Logger.Debugf(m.opts.Context, "Register removed node from service: %s, version: %s", s.Name, s.Version) + m.opts.Logger.Debug(m.opts.Context, fmt.Sprintf("Register removed node from service: %s, version: %s", s.Name, s.Version)) } delete(version.Nodes, n.ID) } @@ -264,7 +265,7 @@ func (m *memory) Deregister(ctx context.Context, s *register.Service, opts ...re go m.sendEvent(®ister.Result{Action: "delete", Service: s}) if m.opts.Logger.V(logger.DebugLevel) { - m.opts.Logger.Debugf(m.opts.Context, "Register removed service: %s", s.Name) + m.opts.Logger.Debug(m.opts.Context, fmt.Sprintf("Register removed service: %s", s.Name)) } return nil } @@ -273,7 +274,7 @@ func (m *memory) Deregister(ctx context.Context, s *register.Service, opts ...re delete(m.records[options.Domain][s.Name], s.Version) go m.sendEvent(®ister.Result{Action: "delete", Service: s}) if m.opts.Logger.V(logger.DebugLevel) { - m.opts.Logger.Debugf(m.opts.Context, "Register removed service: %s, version: %s", s.Name, s.Version) + m.opts.Logger.Debug(m.opts.Context, fmt.Sprintf("Register removed service: %s, version: %s", s.Name, s.Version)) } return nil diff --git a/server/noop.go b/server/noop.go index 6d2ecc05..c4acf493 100644 --- a/server/noop.go +++ b/server/noop.go @@ -459,7 +459,7 @@ func (n *noopServer) Start() error { } } else if rerr != nil && !registered { if config.Logger.V(logger.ErrorLevel) { - config.Logger.Errorf(n.opts.Context, fmt.Sprintf("server %s-%s register check error", config.Name, config.ID), rerr) + config.Logger.Error(n.opts.Context, fmt.Sprintf("server %s-%s register check error", config.Name, config.ID), rerr) } continue } diff --git a/service.go b/service.go index 7a9c3a46..a15897d4 100644 --- a/service.go +++ b/service.go @@ -276,7 +276,7 @@ func (s *service) Start() error { } if config.Loggers[0].V(logger.InfoLevel) { - config.Loggers[0].Infof(s.opts.Context, "starting [service] %s version %s", s.Options().Name, s.Options().Version) + config.Loggers[0].Info(s.opts.Context, fmt.Sprintf("starting [service] %s version %s", s.Options().Name, s.Options().Version)) } if len(s.opts.Servers) == 0 { @@ -322,7 +322,7 @@ func (s *service) Stop() error { s.RUnlock() if config.Loggers[0].V(logger.InfoLevel) { - config.Loggers[0].Infof(s.opts.Context, "stoppping [service] %s", s.Name()) + config.Loggers[0].Info(s.opts.Context, fmt.Sprintf("stoppping [service] %s", s.Name())) } var err error diff --git a/tracer/memory/memory_test.go b/tracer/memory/memory_test.go index 7a6dddd6..bf2b88de 100644 --- a/tracer/memory/memory_test.go +++ b/tracer/memory/memory_test.go @@ -17,14 +17,14 @@ func TestLoggerWithTracer(t *testing.T) { buf := bytes.NewBuffer(nil) logger.DefaultLogger = slog.NewLogger(logger.WithOutput(buf)) - if err := logger.Init(); err != nil { + if err := logger.DefaultLogger.Init(); err != nil { t.Fatal(err) } var span tracer.Span tr := NewTracer() ctx, span = tr.Start(ctx, "test1") - logger.Error(ctx, "my test error", fmt.Errorf("error")) + logger.DefaultLogger.Error(ctx, "my test error", fmt.Errorf("error")) if !strings.Contains(buf.String(), span.TraceID()) { t.Fatalf("log does not contains trace id: %s", buf.Bytes()) diff --git a/util/id/id.go b/util/id/id.go index 2d094f1f..f7d34563 100644 --- a/util/id/id.go +++ b/util/id/id.go @@ -71,7 +71,7 @@ func New(opts ...Option) (string, error) { func Must(opts ...Option) string { id, err := New(opts...) if err != nil { - logger.Fatal(context.TODO(), err) + logger.DefaultLogger.Fatal(context.TODO(), "Must call is failed", err) } return id } From 1fb5673d2778df1abc5b0443d67f56adcf970a23 Mon Sep 17 00:00:00 2001 From: Vasiliy Tolstov Date: Fri, 25 Oct 2024 17:20:35 +0300 Subject: [PATCH 47/59] fixup graceful stop Signed-off-by: Vasiliy Tolstov --- service.go | 7 +++++-- service_test.go | 2 +- 2 files changed, 6 insertions(+), 3 deletions(-) diff --git a/service.go b/service.go index a15897d4..1276c9a3 100644 --- a/service.go +++ b/service.go @@ -86,13 +86,14 @@ func RegisterSubscriber(topic string, s server.Server, h interface{}, opts ...se } type service struct { + done chan struct{} opts Options sync.RWMutex } // NewService creates and returns a new Service based on the packages within. func NewService(opts ...Option) Service { - return &service{opts: NewOptions(opts...)} + return &service{opts: NewOptions(opts...), done: make(chan struct{})} } func (s *service) Name() string { @@ -362,6 +363,8 @@ func (s *service) Stop() error { } } + close(s.done) + return nil } @@ -385,7 +388,7 @@ func (s *service) Run() error { } // wait on context cancel - <-s.opts.Context.Done() + <-s.done return s.Stop() } diff --git a/service_test.go b/service_test.go index b6bca118..7a39c1b5 100644 --- a/service_test.go +++ b/service_test.go @@ -134,7 +134,7 @@ func TestNewService(t *testing.T) { } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { - if got := NewService(tt.args.opts...); !reflect.DeepEqual(got, tt.want) { + if got := NewService(tt.args.opts...); got.Name() != tt.want.Name() { t.Errorf("NewService() = %v, want %v", got.Options().Name, tt.want.Options().Name) } }) From 929e46c0874cf1d8a2cd5b53c18e981f60bd26e6 Mon Sep 17 00:00:00 2001 From: Vasiliy Tolstov Date: Fri, 1 Nov 2024 00:56:40 +0300 Subject: [PATCH 48/59] improve slog Signed-off-by: Vasiliy Tolstov --- logger/slog/slog.go | 60 +++++++++++++++++++++++++++------------- logger/slog/slog_test.go | 9 ++---- 2 files changed, 44 insertions(+), 25 deletions(-) diff --git a/logger/slog/slog.go b/logger/slog/slog.go index 76279083..ee1fe17a 100644 --- a/logger/slog/slog.go +++ b/logger/slog/slog.go @@ -192,27 +192,34 @@ func (s *slogLogger) String() string { return "slog" } -func (s *slogLogger) printLog(ctx context.Context, lvl logger.Level, msg string, attrs ...interface{}) { +func (s *slogLogger) printLog(ctx context.Context, lvl logger.Level, msg string, args ...interface{}) { if !s.V(lvl) { return } + var argError error s.opts.Meter.Counter(semconv.LoggerMessageTotal, "level", lvl.String()).Inc() - attrs = prepareAttributes(attrs) - - for _, fn := range s.opts.ContextAttrFuncs { - a := prepareAttributes(fn(ctx)) - attrs = append(attrs, a...) + attrs, err := s.argsAttrs(args) + if err != nil { + argError = err + } + if argError != nil { + if span, ok := tracer.SpanFromContext(ctx); ok { + span.SetStatus(tracer.SpanStatusError, argError.Error()) + } } - for _, attr := range attrs { - if ve, hasErr := attr.(error); hasErr && ve != nil { - attrs = append(attrs, slog.String(s.opts.ErrorKey, ve.Error())) - if span, ok := tracer.SpanFromContext(ctx); ok { - span.SetStatus(tracer.SpanStatusError, ve.Error()) - } - break + for _, fn := range s.opts.ContextAttrFuncs { + ctxAttrs, err := s.argsAttrs(fn(ctx)) + if err != nil { + argError = err + } + attrs = append(attrs, ctxAttrs...) + } + if argError != nil { + if span, ok := tracer.SpanFromContext(ctx); ok { + span.SetStatus(tracer.SpanStatusError, argError.Error()) } } @@ -229,7 +236,7 @@ func (s *slogLogger) printLog(ctx context.Context, lvl logger.Level, msg string, var pcs [1]uintptr runtime.Callers(s.opts.CallerSkipCount, pcs[:]) // skip [Callers, printLog, LogLvlMethod] r := slog.NewRecord(s.opts.TimeFunc(), loggerToSlogLevel(lvl), msg, pcs[0]) - r.Add(attrs...) + r.AddAttrs(attrs...) _ = s.handler.Handle(ctx, r) } @@ -276,11 +283,26 @@ func slogToLoggerLevel(level slog.Level) logger.Level { } } -func prepareAttributes(attrs []interface{}) []interface{} { - if len(attrs)%2 == 1 { - attrs = append(attrs, badKey) - attrs[len(attrs)-1], attrs[len(attrs)-2] = attrs[len(attrs)-2], attrs[len(attrs)-1] +func (s *slogLogger) argsAttrs(args []interface{}) ([]slog.Attr, error) { + attrs := make([]slog.Attr, 0, len(args)) + var err error + + for idx := 0; idx < len(args); idx++ { + switch arg := args[idx].(type) { + case slog.Attr: + attrs = append(attrs, arg) + case string: + if idx+1 < len(args) { + attrs = append(attrs, slog.Any(arg, args[idx+1])) + idx += 1 + } else { + attrs = append(attrs, slog.String(badKey, arg)) + } + case error: + attrs = append(attrs, slog.String(s.opts.ErrorKey, arg.Error())) + err = arg + } } - return attrs + return attrs, err } diff --git a/logger/slog/slog_test.go b/logger/slog/slog_test.go index f8daace0..fc381892 100644 --- a/logger/slog/slog_test.go +++ b/logger/slog/slog_test.go @@ -5,12 +5,13 @@ import ( "context" "errors" "fmt" - "github.com/google/uuid" - "go.unistack.org/micro/v3/metadata" "log" "strings" "testing" + "github.com/google/uuid" + "go.unistack.org/micro/v3/metadata" + "go.unistack.org/micro/v3/logger" ) @@ -43,9 +44,6 @@ func TestErrorf(t *testing.T) { } l.Log(ctx, logger.ErrorLevel, "message", errors.New("error msg")) - if !bytes.Contains(buf.Bytes(), []byte(`"!BADKEY":"`)) { - t.Fatalf("logger BADKEY not works, buf contains: %s", buf.Bytes()) - } l.Log(ctx, logger.ErrorLevel, "", errors.New("error msg")) if !bytes.Contains(buf.Bytes(), []byte(`"error":"error msg"`)) { @@ -236,5 +234,4 @@ func Test_WithContextAttrFunc(t *testing.T) { if !(bytes.Contains(buf.Bytes(), []byte(`"source-service":"Test-System"`))) { t.Fatalf("logger info, buf %s", buf.Bytes()) } - } From 77eb5b5264c562960f39a75871a298014acfb89c Mon Sep 17 00:00:00 2001 From: Vasiliy Tolstov Date: Fri, 1 Nov 2024 11:23:29 +0300 Subject: [PATCH 49/59] add yaml support Signed-off-by: Vasiliy Tolstov --- util/time/duration.go | 27 +++++++++++++++++++++++++++ util/time/duration_test.go | 36 ++++++++++++++++++++++++++++++++++++ 2 files changed, 63 insertions(+) diff --git a/util/time/duration.go b/util/time/duration.go index 36ba97a1..49550e5e 100644 --- a/util/time/duration.go +++ b/util/time/duration.go @@ -6,6 +6,8 @@ import ( "fmt" "strconv" "time" + + "gopkg.in/yaml.v3" ) type Duration int64 @@ -53,6 +55,31 @@ loop: return time.ParseDuration(fmt.Sprintf("%dh%s", hours, s[p:])) } +func (d Duration) MarshalYAML() (interface{}, error) { + return time.Duration(d).String(), nil +} + +func (d *Duration) UnmarshalYAML(n *yaml.Node) error { + var v interface{} + if err := yaml.Unmarshal([]byte(n.Value), &v); err != nil { + return err + } + switch value := v.(type) { + case float64: + *d = Duration(time.Duration(value)) + return nil + case string: + dv, err := ParseDuration(value) + if err != nil { + return err + } + *d = Duration(dv) + return nil + default: + return fmt.Errorf("invalid duration") + } +} + func (d Duration) MarshalJSON() ([]byte, error) { return json.Marshal(time.Duration(d).String()) } diff --git a/util/time/duration_test.go b/util/time/duration_test.go index 12447510..0324e68a 100644 --- a/util/time/duration_test.go +++ b/util/time/duration_test.go @@ -5,8 +5,44 @@ import ( "encoding/json" "testing" "time" + + "gopkg.in/yaml.v3" ) +func TestMarshalYAML(t *testing.T) { + d := Duration(10000000) + buf, err := yaml.Marshal(d) + if err != nil { + t.Fatal(err) + } + if !bytes.Equal(buf, []byte(`10ms +`)) { + t.Fatalf("invalid duration: %s != %s", buf, `10ms`) + } +} + +func TestUnmarshalYAML(t *testing.T) { + type str struct { + TTL Duration `yaml:"ttl"` + } + v := &str{} + var err error + + err = yaml.Unmarshal([]byte(`{"ttl":"10ms"}`), v) + if err != nil { + t.Fatal(err) + } else if v.TTL != 10000000 { + t.Fatalf("invalid duration %v != 10000000", v.TTL) + } + + err = yaml.Unmarshal([]byte(`{"ttl":"1y"}`), v) + if err != nil { + t.Fatal(err) + } else if v.TTL != 31622400000000000 { + t.Fatalf("invalid duration %v != 31622400000000000", v.TTL) + } +} + func TestMarshalJSON(t *testing.T) { d := Duration(10000000) buf, err := json.Marshal(d) From 7cd7fb0c0a1282cc3329ff72158668b8d0fd905b Mon Sep 17 00:00:00 2001 From: Vasiliy Tolstov Date: Wed, 20 Nov 2024 22:35:36 +0300 Subject: [PATCH 50/59] disable logging for automaxprocs Signed-off-by: Vasiliy Tolstov --- service.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/service.go b/service.go index 1276c9a3..cc827492 100644 --- a/service.go +++ b/service.go @@ -6,7 +6,7 @@ import ( "sync" "github.com/KimMachineGun/automemlimit/memlimit" - _ "go.uber.org/automaxprocs" + "go.uber.org/automaxprocs/maxprocs" "go.unistack.org/micro/v3/broker" "go.unistack.org/micro/v3/client" "go.unistack.org/micro/v3/config" @@ -20,6 +20,7 @@ import ( ) func init() { + maxprocs.Set() memlimit.SetGoMemLimitWithOpts( memlimit.WithRatio(0.9), memlimit.WithProvider( From d1c6e121c16788082f36673a68cf8217c7b5ee75 Mon Sep 17 00:00:00 2001 From: Vasiliy Tolstov Date: Sun, 24 Nov 2024 15:31:40 +0300 Subject: [PATCH 51/59] logger/slog: fix Clone and Fields methods Signed-off-by: Vasiliy Tolstov --- logger/slog/slog.go | 26 ++++++------------ logger/slog/slog_test.go | 57 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 65 insertions(+), 18 deletions(-) diff --git a/logger/slog/slog.go b/logger/slog/slog.go index ee1fe17a..c851ccdc 100644 --- a/logger/slog/slog.go +++ b/logger/slog/slog.go @@ -76,24 +76,19 @@ type slogLogger struct { func (s *slogLogger) Clone(opts ...logger.Option) logger.Logger { s.mu.RLock() options := s.opts + level := s.leveler.Level() s.mu.RUnlock() for _, o := range opts { o(&options) } - l := &slogLogger{ - opts: options, - } + l := &slogLogger{opts: options} l.leveler = new(slog.LevelVar) - handleOpt := &slog.HandlerOptions{ - ReplaceAttr: l.renameAttr, - Level: l.leveler, - AddSource: l.opts.AddSource, - } - l.leveler.Set(loggerToSlogLevel(l.opts.Level)) - l.handler = slog.New(slog.NewJSONHandler(options.Out, handleOpt)).With(options.Fields...).Handler() + l.leveler.Set(level) + attrs, _ := s.argsAttrs(l.opts.Fields) + l.handler = s.handler.WithAttrs(attrs) return l } @@ -110,7 +105,7 @@ func (s *slogLogger) Options() logger.Options { return s.opts } -func (s *slogLogger) Fields(attrs ...interface{}) logger.Logger { +func (s *slogLogger) Fields(fields ...interface{}) logger.Logger { s.mu.RLock() level := s.leveler.Level() options := s.opts @@ -120,13 +115,8 @@ func (s *slogLogger) Fields(attrs ...interface{}) logger.Logger { l.leveler = new(slog.LevelVar) l.leveler.Set(level) - handleOpt := &slog.HandlerOptions{ - ReplaceAttr: l.renameAttr, - Level: l.leveler, - AddSource: l.opts.AddSource, - } - - l.handler = slog.New(slog.NewJSONHandler(l.opts.Out, handleOpt)).With(attrs...).Handler() + attrs, _ := s.argsAttrs(fields) + l.handler = s.handler.WithAttrs(attrs) return l } diff --git a/logger/slog/slog_test.go b/logger/slog/slog_test.go index fc381892..d27de0f0 100644 --- a/logger/slog/slog_test.go +++ b/logger/slog/slog_test.go @@ -15,6 +15,56 @@ import ( "go.unistack.org/micro/v3/logger" ) +func TestMultipleFieldsWithLevel(t *testing.T) { + ctx := context.TODO() + buf := bytes.NewBuffer(nil) + l := NewLogger(logger.WithLevel(logger.InfoLevel), logger.WithOutput(buf)) + if err := l.Init(); err != nil { + t.Fatal(err) + } + + l = l.Fields("key", "val") + + l.Info(ctx, "msg1") + nl := l.Clone(logger.WithLevel(logger.DebugLevel)) + nl.Debug(ctx, "msg2") + l.Debug(ctx, "msg3") + if !bytes.Contains(buf.Bytes(), []byte(`"key":"val"`)) { + t.Fatalf("logger error not works, buf contains: %s", buf.Bytes()) + } + if !bytes.Contains(buf.Bytes(), []byte(`"msg1"`)) { + t.Fatalf("logger error not works, buf contains: %s", buf.Bytes()) + } + if !bytes.Contains(buf.Bytes(), []byte(`"msg2"`)) { + t.Fatalf("logger error not works, buf contains: %s", buf.Bytes()) + } + if bytes.Contains(buf.Bytes(), []byte(`"msg3"`)) { + t.Fatalf("logger error not works, buf contains: %s", buf.Bytes()) + } +} + +func TestMultipleFields(t *testing.T) { + ctx := context.TODO() + buf := bytes.NewBuffer(nil) + l := NewLogger(logger.WithLevel(logger.InfoLevel), logger.WithOutput(buf)) + if err := l.Init(); err != nil { + t.Fatal(err) + } + + l = l.Fields("key", "val") + + l = l.Fields("key1", "val1") + + l.Info(ctx, "msg") + + if !bytes.Contains(buf.Bytes(), []byte(`"key":"val"`)) { + t.Fatalf("logger error not works, buf contains: %s", buf.Bytes()) + } + if !bytes.Contains(buf.Bytes(), []byte(`"key1":"val1"`)) { + t.Fatalf("logger error not works, buf contains: %s", buf.Bytes()) + } +} + func TestError(t *testing.T) { ctx := context.TODO() buf := bytes.NewBuffer(nil) @@ -234,4 +284,11 @@ func Test_WithContextAttrFunc(t *testing.T) { if !(bytes.Contains(buf.Bytes(), []byte(`"source-service":"Test-System"`))) { t.Fatalf("logger info, buf %s", buf.Bytes()) } + buf.Reset() + imd, _ := metadata.FromIncomingContext(ctx) + l.Info(ctx, "test message1") + imd.Set("Source-Service", "Test-System2") + l.Info(ctx, "test message2") + + // t.Logf("xxx %s", buf.Bytes()) } From 49055a28eaab78c0cdd3fe724113d737eea3b4a0 Mon Sep 17 00:00:00 2001 From: Vasiliy Tolstov Date: Sun, 24 Nov 2024 23:28:15 +0300 Subject: [PATCH 52/59] logger/slog: wrap handler Signed-off-by: Vasiliy Tolstov --- logger/slog/slog.go | 67 +++++++++++++++++++++++++++++++-------------- 1 file changed, 47 insertions(+), 20 deletions(-) diff --git a/logger/slog/slog.go b/logger/slog/slog.go index c851ccdc..10a40533 100644 --- a/logger/slog/slog.go +++ b/logger/slog/slog.go @@ -31,6 +31,28 @@ var ( fatalValue = slog.StringValue("fatal") ) +type wrapper struct { + h slog.Handler + level logger.Level +} + +func (h *wrapper) Enabled(ctx context.Context, level slog.Level) bool { + lvl := slogToLoggerLevel(level) + return h.level.Enabled(lvl) +} + +func (h *wrapper) Handle(ctx context.Context, rec slog.Record) error { + return h.h.Handle(ctx, rec) +} + +func (h *wrapper) WithAttrs(attrs []slog.Attr) slog.Handler { + return h.WithAttrs(attrs) +} + +func (h *wrapper) WithGroup(name string) slog.Handler { + return h.WithGroup(name) +} + func (s *slogLogger) renameAttr(_ []string, a slog.Attr) slog.Attr { switch a.Key { case slog.SourceKey: @@ -68,7 +90,7 @@ func (s *slogLogger) renameAttr(_ []string, a slog.Attr) slog.Attr { type slogLogger struct { leveler *slog.LevelVar - handler slog.Handler + handler *wrapper opts logger.Options mu sync.RWMutex } @@ -76,19 +98,21 @@ type slogLogger struct { func (s *slogLogger) Clone(opts ...logger.Option) logger.Logger { s.mu.RLock() options := s.opts - level := s.leveler.Level() s.mu.RUnlock() for _, o := range opts { o(&options) } - l := &slogLogger{opts: options} + if len(options.ContextAttrFuncs) == 0 { + options.ContextAttrFuncs = logger.DefaultContextAttrFuncs + } - l.leveler = new(slog.LevelVar) - l.leveler.Set(level) - attrs, _ := s.argsAttrs(l.opts.Fields) - l.handler = s.handler.WithAttrs(attrs) + attrs, _ := s.argsAttrs(options.Fields) + l := &slogLogger{ + handler: &wrapper{level: options.Level, h: s.handler.h.WithAttrs(attrs)}, + opts: options, + } return l } @@ -98,7 +122,9 @@ func (s *slogLogger) V(level logger.Level) bool { } func (s *slogLogger) Level(level logger.Level) { - s.leveler.Set(loggerToSlogLevel(level)) + s.mu.Lock() + s.opts.Level = level + s.mu.Unlock() } func (s *slogLogger) Options() logger.Options { @@ -107,16 +133,17 @@ func (s *slogLogger) Options() logger.Options { func (s *slogLogger) Fields(fields ...interface{}) logger.Logger { s.mu.RLock() - level := s.leveler.Level() options := s.opts s.mu.RUnlock() l := &slogLogger{opts: options} - l.leveler = new(slog.LevelVar) - l.leveler.Set(level) + + if len(options.ContextAttrFuncs) == 0 { + options.ContextAttrFuncs = logger.DefaultContextAttrFuncs + } attrs, _ := s.argsAttrs(fields) - l.handler = s.handler.WithAttrs(attrs) + l.handler = &wrapper{level: s.opts.Level, h: s.handler.h.WithAttrs(attrs)} return l } @@ -124,22 +151,22 @@ func (s *slogLogger) Fields(fields ...interface{}) logger.Logger { func (s *slogLogger) Init(opts ...logger.Option) error { s.mu.Lock() - if len(s.opts.ContextAttrFuncs) == 0 { - s.opts.ContextAttrFuncs = logger.DefaultContextAttrFuncs - } - for _, o := range opts { o(&s.opts) } - s.leveler = new(slog.LevelVar) + if len(s.opts.ContextAttrFuncs) == 0 { + s.opts.ContextAttrFuncs = logger.DefaultContextAttrFuncs + } + handleOpt := &slog.HandlerOptions{ ReplaceAttr: s.renameAttr, - Level: s.leveler, + Level: loggerToSlogLevel(logger.TraceLevel), AddSource: s.opts.AddSource, } - s.leveler.Set(loggerToSlogLevel(s.opts.Level)) - s.handler = slog.New(slog.NewJSONHandler(s.opts.Out, handleOpt)).With(s.opts.Fields...).Handler() + + attrs, _ := s.argsAttrs(s.opts.Fields) + s.handler = &wrapper{level: s.opts.Level, h: slog.NewJSONHandler(s.opts.Out, handleOpt).WithAttrs(attrs)} s.mu.Unlock() return nil From ad19fe2b90a0b664f9a2cede43bbabd4291b801c Mon Sep 17 00:00:00 2001 From: Vasiliy Tolstov Date: Sun, 24 Nov 2024 23:40:54 +0300 Subject: [PATCH 53/59] logger/slog: fix race condigtion with Enabled and Level Signed-off-by: Vasiliy Tolstov --- logger/slog/slog.go | 21 ++++++++++++++------- 1 file changed, 14 insertions(+), 7 deletions(-) diff --git a/logger/slog/slog.go b/logger/slog/slog.go index 10a40533..4d3e8c06 100644 --- a/logger/slog/slog.go +++ b/logger/slog/slog.go @@ -8,6 +8,7 @@ import ( "runtime" "strconv" "sync" + "sync/atomic" "go.unistack.org/micro/v3/logger" "go.unistack.org/micro/v3/semconv" @@ -33,12 +34,11 @@ var ( type wrapper struct { h slog.Handler - level logger.Level + level atomic.Int64 } func (h *wrapper) Enabled(ctx context.Context, level slog.Level) bool { - lvl := slogToLoggerLevel(level) - return h.level.Enabled(lvl) + return level >= slog.Level(int(h.level.Load())) } func (h *wrapper) Handle(ctx context.Context, rec slog.Record) error { @@ -110,20 +110,25 @@ func (s *slogLogger) Clone(opts ...logger.Option) logger.Logger { attrs, _ := s.argsAttrs(options.Fields) l := &slogLogger{ - handler: &wrapper{level: options.Level, h: s.handler.h.WithAttrs(attrs)}, + handler: &wrapper{h: s.handler.h.WithAttrs(attrs)}, opts: options, } + l.handler.level.Store(int64(loggerToSlogLevel(options.Level))) return l } func (s *slogLogger) V(level logger.Level) bool { - return s.opts.Level.Enabled(level) + s.mu.Lock() + v := s.opts.Level.Enabled(level) + s.mu.Unlock() + return v } func (s *slogLogger) Level(level logger.Level) { s.mu.Lock() s.opts.Level = level + s.handler.level.Store(int64(loggerToSlogLevel(level))) s.mu.Unlock() } @@ -143,7 +148,8 @@ func (s *slogLogger) Fields(fields ...interface{}) logger.Logger { } attrs, _ := s.argsAttrs(fields) - l.handler = &wrapper{level: s.opts.Level, h: s.handler.h.WithAttrs(attrs)} + l.handler = &wrapper{h: s.handler.h.WithAttrs(attrs)} + l.handler.level.Store(int64(loggerToSlogLevel(l.opts.Level))) return l } @@ -166,7 +172,8 @@ func (s *slogLogger) Init(opts ...logger.Option) error { } attrs, _ := s.argsAttrs(s.opts.Fields) - s.handler = &wrapper{level: s.opts.Level, h: slog.NewJSONHandler(s.opts.Out, handleOpt).WithAttrs(attrs)} + s.handler = &wrapper{h: slog.NewJSONHandler(s.opts.Out, handleOpt).WithAttrs(attrs)} + s.handler.level.Store(int64(loggerToSlogLevel(s.opts.Level))) s.mu.Unlock() return nil From 2a6ce6d4da7305aecf6fa56b83ceb1944052fcb2 Mon Sep 17 00:00:00 2001 From: Evstigneev Denis Date: Tue, 26 Nov 2024 12:18:17 +0300 Subject: [PATCH 54/59] add using lazy connect (#361) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit #357 Co-authored-by: Василий Толстов Reviewed-on: https://git.unistack.org/unistack-org/micro/pulls/361 Reviewed-by: Василий Толстов Co-authored-by: Evstigneev Denis Co-committed-by: Evstigneev Denis --- store/memory/memory.go | 51 ++++++++++++++++++++++++++++++------ store/noop.go | 59 ++++++++++++++++++++++++++++++++++-------- store/options.go | 2 ++ 3 files changed, 93 insertions(+), 19 deletions(-) diff --git a/store/memory/memory.go b/store/memory/memory.go index 7f012fab..11b41046 100644 --- a/store/memory/memory.go +++ b/store/memory/memory.go @@ -4,6 +4,7 @@ import ( "context" "sort" "strings" + "sync/atomic" "time" cache "github.com/patrickmn/go-cache" @@ -20,7 +21,10 @@ func NewStore(opts ...store.Option) store.Store { } func (m *memoryStore) Connect(ctx context.Context) error { - return nil + if m.opts.LazyConnect { + return nil + } + return m.connect(ctx) } func (m *memoryStore) Disconnect(ctx context.Context) error { @@ -29,13 +33,14 @@ func (m *memoryStore) Disconnect(ctx context.Context) error { } type memoryStore struct { - funcRead store.FuncRead - funcWrite store.FuncWrite - funcExists store.FuncExists - funcList store.FuncList - funcDelete store.FuncDelete - store *cache.Cache - opts store.Options + funcRead store.FuncRead + funcWrite store.FuncWrite + funcExists store.FuncExists + funcList store.FuncList + funcDelete store.FuncDelete + store *cache.Cache + opts store.Options + isConnected atomic.Int32 } func (m *memoryStore) key(prefix, key string) string { @@ -145,6 +150,11 @@ func (m *memoryStore) Name() string { } func (m *memoryStore) Exists(ctx context.Context, key string, opts ...store.ExistsOption) error { + if m.opts.LazyConnect { + if err := m.connect(ctx); err != nil { + return err + } + } return m.funcExists(ctx, key, opts...) } @@ -157,6 +167,11 @@ func (m *memoryStore) fnExists(ctx context.Context, key string, opts ...store.Ex } func (m *memoryStore) Read(ctx context.Context, key string, val interface{}, opts ...store.ReadOption) error { + if m.opts.LazyConnect { + if err := m.connect(ctx); err != nil { + return err + } + } return m.funcRead(ctx, key, val, opts...) } @@ -169,6 +184,11 @@ func (m *memoryStore) fnRead(ctx context.Context, key string, val interface{}, o } func (m *memoryStore) Write(ctx context.Context, key string, val interface{}, opts ...store.WriteOption) error { + if m.opts.LazyConnect { + if err := m.connect(ctx); err != nil { + return err + } + } return m.funcWrite(ctx, key, val, opts...) } @@ -193,6 +213,11 @@ func (m *memoryStore) fnWrite(ctx context.Context, key string, val interface{}, } func (m *memoryStore) Delete(ctx context.Context, key string, opts ...store.DeleteOption) error { + if m.opts.LazyConnect { + if err := m.connect(ctx); err != nil { + return err + } + } return m.funcDelete(ctx, key, opts...) } @@ -211,6 +236,11 @@ func (m *memoryStore) Options() store.Options { } func (m *memoryStore) List(ctx context.Context, opts ...store.ListOption) ([]string, error) { + if m.opts.LazyConnect { + if err := m.connect(ctx); err != nil { + return nil, err + } + } return m.funcList(ctx, opts...) } @@ -244,3 +274,8 @@ func (m *memoryStore) fnList(ctx context.Context, opts ...store.ListOption) ([]s return keys, nil } + +func (m *memoryStore) connect(ctx context.Context) error { + m.isConnected.CompareAndSwap(0, 1) + return nil +} diff --git a/store/noop.go b/store/noop.go index c7e2922d..7b113dfa 100644 --- a/store/noop.go +++ b/store/noop.go @@ -2,6 +2,7 @@ package store import ( "context" + "sync/atomic" "go.unistack.org/micro/v3/options" ) @@ -9,12 +10,13 @@ import ( var _ Store = (*noopStore)(nil) type noopStore struct { - funcRead FuncRead - funcWrite FuncWrite - funcExists FuncExists - funcList FuncList - funcDelete FuncDelete - opts Options + funcRead FuncRead + funcWrite FuncWrite + funcExists FuncExists + funcList FuncList + funcDelete FuncDelete + opts Options + isConnected atomic.Int32 } func NewStore(opts ...Option) *noopStore { @@ -52,12 +54,10 @@ func (n *noopStore) Init(opts ...Option) error { } func (n *noopStore) Connect(ctx context.Context) error { - select { - case <-ctx.Done(): - return ctx.Err() - default: + if n.opts.LazyConnect { + return nil } - return nil + return n.connect(ctx) } func (n *noopStore) Disconnect(ctx context.Context) error { @@ -70,6 +70,11 @@ func (n *noopStore) Disconnect(ctx context.Context) error { } func (n *noopStore) Read(ctx context.Context, key string, val interface{}, opts ...ReadOption) error { + if n.opts.LazyConnect { + if err := n.connect(ctx); err != nil { + return err + } + } return n.funcRead(ctx, key, val, opts...) } @@ -83,6 +88,11 @@ func (n *noopStore) fnRead(ctx context.Context, key string, val interface{}, opt } func (n *noopStore) Delete(ctx context.Context, key string, opts ...DeleteOption) error { + if n.opts.LazyConnect { + if err := n.connect(ctx); err != nil { + return err + } + } return n.funcDelete(ctx, key, opts...) } @@ -96,6 +106,11 @@ func (n *noopStore) fnDelete(ctx context.Context, key string, opts ...DeleteOpti } func (n *noopStore) Exists(ctx context.Context, key string, opts ...ExistsOption) error { + if n.opts.LazyConnect { + if err := n.connect(ctx); err != nil { + return err + } + } return n.funcExists(ctx, key, opts...) } @@ -109,6 +124,11 @@ func (n *noopStore) fnExists(ctx context.Context, key string, opts ...ExistsOpti } func (n *noopStore) Write(ctx context.Context, key string, val interface{}, opts ...WriteOption) error { + if n.opts.LazyConnect { + if err := n.connect(ctx); err != nil { + return err + } + } return n.funcWrite(ctx, key, val, opts...) } @@ -122,6 +142,11 @@ func (n *noopStore) fnWrite(ctx context.Context, key string, val interface{}, op } func (n *noopStore) List(ctx context.Context, opts ...ListOption) ([]string, error) { + if n.opts.LazyConnect { + if err := n.connect(ctx); err != nil { + return nil, err + } + } return n.funcList(ctx, opts...) } @@ -145,3 +170,15 @@ func (n *noopStore) String() string { func (n *noopStore) Options() Options { return n.opts } + +func (n *noopStore) connect(ctx context.Context) error { + if n.isConnected.CompareAndSwap(0, 1) { + select { + case <-ctx.Done(): + return ctx.Err() + default: + } + } + + return nil +} diff --git a/store/options.go b/store/options.go index 438b745c..ffcafdd5 100644 --- a/store/options.go +++ b/store/options.go @@ -41,6 +41,8 @@ type Options struct { Timeout time.Duration // Hooks can be run before/after store Read/List/Write/Exists/Delete Hooks options.Hooks + // LazyConnect creates a connection when using store + LazyConnect bool } // NewOptions creates options struct From 89cf4ef8af544bfda2f41fdcefa91b61fad781ef Mon Sep 17 00:00:00 2001 From: Vasiliy Tolstov Date: Tue, 26 Nov 2024 17:48:09 +0300 Subject: [PATCH 55/59] store: add missin LazyConnect option Signed-off-by: Vasiliy Tolstov --- store/options.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/store/options.go b/store/options.go index ffcafdd5..a4871cd1 100644 --- a/store/options.go +++ b/store/options.go @@ -134,6 +134,13 @@ func Timeout(td time.Duration) Option { } } +// LazyConnect initialize connection only when needed +func LazyConnect(b bool) Option { + return func(o *Options) { + o.LazyConnect = b + } +} + // Addrs contains the addresses or other connection information of the backing storage. // For example, an etcd implementation would contain the nodes of the cluster. // A SQL implementation could contain one or more connection strings. From 115ca6a018470a23f891097f2e440838119a3c86 Mon Sep 17 00:00:00 2001 From: Vasiliy Tolstov Date: Fri, 29 Nov 2024 15:34:02 +0300 Subject: [PATCH 56/59] logger: add WithAddFields option Signed-off-by: Vasiliy Tolstov --- logger/options.go | 7 +++++++ logger/slog/slog_test.go | 28 ++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+) diff --git a/logger/options.go b/logger/options.go index ede9a23b..a1a71048 100644 --- a/logger/options.go +++ b/logger/options.go @@ -80,6 +80,13 @@ func WithContextAttrFuncs(fncs ...ContextAttrFunc) Option { } } +// WithAddFields add fields for the logger +func WithAddFields(fields ...interface{}) Option { + return func(o *Options) { + o.Fields = append(o.Fields, fields...) + } +} + // WithFields set default fields for the logger func WithFields(fields ...interface{}) Option { return func(o *Options) { diff --git a/logger/slog/slog_test.go b/logger/slog/slog_test.go index d27de0f0..ba9b69ba 100644 --- a/logger/slog/slog_test.go +++ b/logger/slog/slog_test.go @@ -15,6 +15,34 @@ import ( "go.unistack.org/micro/v3/logger" ) +func TestWithAddFields(t *testing.T) { + ctx := context.TODO() + buf := bytes.NewBuffer(nil) + l := NewLogger(logger.WithLevel(logger.InfoLevel), logger.WithOutput(buf)) + if err := l.Init(); err != nil { + t.Fatal(err) + } + + l.Info(ctx, "msg1") + + if err := l.Init(logger.WithAddFields("key1", "val1")); err != nil { + t.Fatal(err) + } + l.Info(ctx, "msg2") + + if err := l.Init(logger.WithAddFields("key2", "val2")); err != nil { + t.Fatal(err) + } + l.Info(ctx, "msg3") + + if !bytes.Contains(buf.Bytes(), []byte(`"key1"`)) { + t.Fatalf("logger error not works, buf contains: %s", buf.Bytes()) + } + if !bytes.Contains(buf.Bytes(), []byte(`"key2"`)) { + t.Fatalf("logger error not works, buf contains: %s", buf.Bytes()) + } +} + func TestMultipleFieldsWithLevel(t *testing.T) { ctx := context.TODO() buf := bytes.NewBuffer(nil) From ae97023092894aee1378401e0dee73b01d3e59cb Mon Sep 17 00:00:00 2001 From: Vasiliy Tolstov Date: Sun, 1 Dec 2024 19:54:38 +0300 Subject: [PATCH 57/59] store: updates for Watcher Signed-off-by: Vasiliy Tolstov --- store/noop.go | 42 ++++++++++++++++++++++++++++++++++++++++++ store/store.go | 41 +++++++++++++++++++++++++++++++++++++++++ store/wrapper.go | 14 ++------------ 3 files changed, 85 insertions(+), 12 deletions(-) diff --git a/store/noop.go b/store/noop.go index 7b113dfa..83524901 100644 --- a/store/noop.go +++ b/store/noop.go @@ -2,14 +2,18 @@ package store import ( "context" + "sync" "sync/atomic" "go.unistack.org/micro/v3/options" + "go.unistack.org/micro/v3/util/id" ) var _ Store = (*noopStore)(nil) type noopStore struct { + mu sync.Mutex + watchers map[string]Watcher funcRead FuncRead funcWrite FuncWrite funcExists FuncExists @@ -182,3 +186,41 @@ func (n *noopStore) connect(ctx context.Context) error { return nil } + +type watcher struct { + exit chan bool + id string + ch chan Event + opts WatchOptions +} + +func (m *noopStore) Watch(ctx context.Context, opts ...WatchOption) (Watcher, error) { + id, err := id.New() + if err != nil { + return nil, err + } + wo, err := NewWatchOptions(opts...) + if err != nil { + return nil, err + } + // construct the watcher + w := &watcher{ + exit: make(chan bool), + ch: make(chan Event), + id: id, + opts: wo, + } + + m.mu.Lock() + m.watchers[w.id] = w + m.mu.Unlock() + + return w, nil +} + +func (w *watcher) Next() (Event, error) { + return nil, nil +} + +func (w *watcher) Stop() { +} diff --git a/store/store.go b/store/store.go index 25530cc4..8235be5d 100644 --- a/store/store.go +++ b/store/store.go @@ -4,9 +4,11 @@ package store import ( "context" "errors" + "time" ) var ( + ErrWatcherStopped = errors.New("watcher stopped") // ErrNotConnected is returned when a store is not connected ErrNotConnected = errors.New("not conected") // ErrNotFound is returned when a key doesn't exist @@ -43,6 +45,7 @@ type Store interface { Disconnect(ctx context.Context) error // String returns the name of the implementation. String() string + Watch(ctx context.Context, opts ...WatchOption) (Watcher, error) } type ( @@ -57,3 +60,41 @@ type ( FuncList func(ctx context.Context, opts ...ListOption) ([]string, error) HookList func(next FuncList) FuncList ) + +type EventType int + +const ( + EventTypeUnknown = iota + EventTypeConnect + EventTypeDisconnect + EventTypeOpError +) + +type Event interface { + Timestamp() time.Time + Error() error + Type() EventType +} + +type Watcher interface { + // Next is a blocking call + Next() (Event, error) + // Stop stops the watcher + Stop() +} + +type WatchOption func(*WatchOptions) error + +type WatchOptions struct{} + +func NewWatchOptions(opts ...WatchOption) (WatchOptions, error) { + options := WatchOptions{} + var err error + for _, o := range opts { + if err = o(&options); err != nil { + break + } + } + + return options, err +} diff --git a/store/wrapper.go b/store/wrapper.go index 84c43026..a9221bae 100644 --- a/store/wrapper.go +++ b/store/wrapper.go @@ -67,16 +67,6 @@ func (w *NamespaceStore) String() string { return w.s.String() } -// type NamespaceWrapper struct{} - -// func NewNamespaceWrapper() Wrapper { -// return &NamespaceWrapper{} -// } - -/* -func (w *OmitWrapper) Logf(fn LogfFunc) LogfFunc { - return func(ctx context.Context, level Level, msg string, args ...interface{}) { - fn(ctx, level, msg, getArgs(args)...) - } +func (w *NamespaceStore) Watch(ctx context.Context, opts ...WatchOption) (Watcher, error) { + return w.s.Watch(ctx, opts...) } -*/ From 36b7b9f5fb3968f01dc66f80dbfff93bdc0a68a8 Mon Sep 17 00:00:00 2001 From: Vasiliy Tolstov Date: Mon, 2 Dec 2024 13:20:13 +0300 Subject: [PATCH 58/59] add Live/Ready/Health methods Signed-off-by: Vasiliy Tolstov --- broker/broker.go | 6 ++ broker/memory/memory.go | 12 ++++ broker/noop.go | 12 ++++ client/options.go | 45 +++--------- cluster/cluster.go | 6 ++ config/config.go | 13 ++-- metadata/metadata_test.go | 22 ++++++ micro_test.go | 6 ++ network/tunnel/broker/broker.go | 12 ++++ register/register.go | 15 ++++ server/noop.go | 12 ++++ server/options.go | 19 ----- server/server.go | 6 ++ service.go | 75 +++++++++++++++++--- store/memory/memory.go | 25 +++++++ store/noop.go | 12 ++++ store/store.go | 7 ++ store/wrapper.go | 12 ++++ util/io/io.go | 40 ----------- util/pool/default.go | 118 -------------------------------- util/pool/default_test.go | 92 ------------------------- util/pool/options.go | 38 ---------- util/pool/pool.go | 38 ---------- util/time/duration_test.go | 6 +- 24 files changed, 249 insertions(+), 400 deletions(-) delete mode 100644 util/io/io.go delete mode 100644 util/pool/default.go delete mode 100644 util/pool/default_test.go delete mode 100644 util/pool/options.go delete mode 100644 util/pool/pool.go diff --git a/broker/broker.go b/broker/broker.go index a67a0be8..adf4ed4a 100644 --- a/broker/broker.go +++ b/broker/broker.go @@ -46,6 +46,12 @@ type Broker interface { BatchSubscribe(ctx context.Context, topic string, h BatchHandler, opts ...SubscribeOption) (Subscriber, error) // String type of broker String() string + // Live returns broker liveness + Live() bool + // Ready returns broker readiness + Ready() bool + // Health returns broker health + Health() bool } type ( diff --git a/broker/memory/memory.go b/broker/memory/memory.go index de8f1c73..56c08eaa 100644 --- a/broker/memory/memory.go +++ b/broker/memory/memory.go @@ -339,6 +339,18 @@ func (m *memoryBroker) Name() string { return m.opts.Name } +func (m *memoryBroker) Live() bool { + return true +} + +func (m *memoryBroker) Ready() bool { + return true +} + +func (m *memoryBroker) Health() bool { + return true +} + func (m *memoryEvent) Topic() string { return m.topic } diff --git a/broker/noop.go b/broker/noop.go index 43f6e7dc..de98421e 100644 --- a/broker/noop.go +++ b/broker/noop.go @@ -25,6 +25,18 @@ func NewBroker(opts ...Option) *NoopBroker { return b } +func (b *NoopBroker) Health() bool { + return true +} + +func (b *NoopBroker) Live() bool { + return true +} + +func (b *NoopBroker) Ready() bool { + return true +} + func (b *NoopBroker) Name() string { return b.opts.Name } diff --git a/client/options.go b/client/options.go index 6c8d0637..84b86672 100644 --- a/client/options.go +++ b/client/options.go @@ -11,7 +11,6 @@ import ( "go.unistack.org/micro/v3/logger" "go.unistack.org/micro/v3/metadata" "go.unistack.org/micro/v3/meter" - "go.unistack.org/micro/v3/network/transport" "go.unistack.org/micro/v3/options" "go.unistack.org/micro/v3/register" "go.unistack.org/micro/v3/router" @@ -22,8 +21,6 @@ import ( // Options holds client options type Options struct { - // Transport used for transfer messages - Transport transport.Transport // Selector used to select needed address Selector selector.Selector // Logger used to log messages @@ -194,18 +191,16 @@ func NewOptions(opts ...Option) Options { Retry: DefaultRetry, Retries: DefaultRetries, RequestTimeout: DefaultRequestTimeout, - DialTimeout: transport.DefaultDialTimeout, }, - Lookup: LookupRoute, - PoolSize: DefaultPoolSize, - PoolTTL: DefaultPoolTTL, - Selector: random.NewSelector(), - Logger: logger.DefaultLogger, - Broker: broker.DefaultBroker, - Meter: meter.DefaultMeter, - Tracer: tracer.DefaultTracer, - Router: router.DefaultRouter, - Transport: transport.DefaultTransport, + Lookup: LookupRoute, + PoolSize: DefaultPoolSize, + PoolTTL: DefaultPoolTTL, + Selector: random.NewSelector(), + Logger: logger.DefaultLogger, + Broker: broker.DefaultBroker, + Meter: meter.DefaultMeter, + Tracer: tracer.DefaultTracer, + Router: router.DefaultRouter, } for _, o := range opts { @@ -278,13 +273,6 @@ func PoolTTL(d time.Duration) Option { } } -// Transport to use for communication e.g http, rabbitmq, etc -func Transport(t transport.Transport) Option { - return func(o *Options) { - o.Transport = t - } -} - // Register sets the routers register func Register(r register.Register) Option { return func(o *Options) { @@ -334,14 +322,6 @@ func TLSConfig(t *tls.Config) Option { return func(o *Options) { // set the internal tls o.TLSConfig = t - - // set the default transport if one is not - // already set. Required for Init call below. - - // set the transport tls - _ = o.Transport.Init( - transport.TLSConfig(t), - ) } } @@ -507,13 +487,6 @@ func WithAuthToken(t string) CallOption { } } -// WithNetwork is a CallOption which sets the network attribute -func WithNetwork(n string) CallOption { - return func(o *CallOptions) { - o.Network = n - } -} - // WithRouter sets the router to use for this call func WithRouter(r router.Router) CallOption { return func(o *CallOptions) { diff --git a/cluster/cluster.go b/cluster/cluster.go index 26549e62..01544eb8 100644 --- a/cluster/cluster.go +++ b/cluster/cluster.go @@ -38,4 +38,10 @@ type Cluster interface { Broadcast(ctx context.Context, msg Message, filter ...string) error // Unicast send message to single member in cluster Unicast(ctx context.Context, node Node, msg Message) error + // Live returns cluster liveness + Live() bool + // Ready returns cluster readiness + Ready() bool + // Health returns cluster health + Health() bool } diff --git a/config/config.go b/config/config.go index 0ca629e4..a1831826 100644 --- a/config/config.go +++ b/config/config.go @@ -4,7 +4,6 @@ package config import ( "context" "errors" - "fmt" "reflect" "time" ) @@ -139,7 +138,7 @@ var ( return nil } if err := fn(ctx, c); err != nil { - c.Options().Logger.Error(ctx, fmt.Sprintf("%s BeforeLoad error", c.String()), err) + c.Options().Logger.Error(ctx, c.String()+" BeforeLoad error", err) if !c.Options().AllowFail { return err } @@ -154,7 +153,7 @@ var ( return nil } if err := fn(ctx, c); err != nil { - c.Options().Logger.Error(ctx, fmt.Sprintf("%s AfterLoad error", c.String()), err) + c.Options().Logger.Error(ctx, c.String()+" AfterLoad error", err) if !c.Options().AllowFail { return err } @@ -169,7 +168,7 @@ var ( return nil } if err := fn(ctx, c); err != nil { - c.Options().Logger.Error(ctx, fmt.Sprintf("%s BeforeSave error", c.String()), err) + c.Options().Logger.Error(ctx, c.String()+" BeforeSave error", err) if !c.Options().AllowFail { return err } @@ -184,7 +183,7 @@ var ( return nil } if err := fn(ctx, c); err != nil { - c.Options().Logger.Error(ctx, fmt.Sprintf("%s AfterSave error", c.String()), err) + c.Options().Logger.Error(ctx, c.String()+" AfterSave error", err) if !c.Options().AllowFail { return err } @@ -199,7 +198,7 @@ var ( return nil } if err := fn(ctx, c); err != nil { - c.Options().Logger.Error(ctx, fmt.Sprintf("%s BeforeInit error", c.String()), err) + c.Options().Logger.Error(ctx, c.String()+" BeforeInit error", err) if !c.Options().AllowFail { return err } @@ -214,7 +213,7 @@ var ( return nil } if err := fn(ctx, c); err != nil { - c.Options().Logger.Error(ctx, fmt.Sprintf("%s AfterInit error", c.String(), err), err) + c.Options().Logger.Error(ctx, c.String()+" AfterInit error", err) if !c.Options().AllowFail { return err } diff --git a/metadata/metadata_test.go b/metadata/metadata_test.go index a5381ede..5aa41133 100644 --- a/metadata/metadata_test.go +++ b/metadata/metadata_test.go @@ -5,6 +5,28 @@ import ( "testing" ) +func TestMultipleUsage(t *testing.T) { + ctx := context.TODO() + md := New(0) + md.Set("key1_1", "val1_1", "key1_2", "val1_2", "key1_3", "val1_3") + ctx = NewIncomingContext(ctx, Copy(md)) + ctx = NewOutgoingContext(ctx, Copy(md)) + imd, _ := FromIncomingContext(ctx) + omd, _ := FromOutgoingContext(ctx) + _ = func(x context.Context) context.Context { + m, _ := FromIncomingContext(x) + m.Del("key1_2") + return ctx + }(ctx) + _ = func(x context.Context) context.Context { + m, _ := FromIncomingContext(x) + m.Del("key1_3") + return ctx + }(ctx) + t.Logf("imd %#+v", imd) + t.Logf("omd %#+v", omd) +} + func TestMetadataSetMultiple(t *testing.T) { md := New(4) md.Set("key1", "val1", "key2", "val2", "key3") diff --git a/micro_test.go b/micro_test.go index d2a20121..20272e8c 100644 --- a/micro_test.go +++ b/micro_test.go @@ -66,6 +66,12 @@ type bro struct { func (p *bro) Name() string { return p.name } +func (p *bro) Live() bool { return true } + +func (p *bro) Ready() bool { return true } + +func (p *bro) Health() bool { return true } + func (p *bro) Init(opts ...broker.Option) error { return nil } // Options returns broker options diff --git a/network/tunnel/broker/broker.go b/network/tunnel/broker/broker.go index 921d1853..a05e0ecb 100644 --- a/network/tunnel/broker/broker.go +++ b/network/tunnel/broker/broker.go @@ -45,6 +45,18 @@ type ( tunnelAddr struct{} ) +func (t *tunBroker) Live() bool { + return true +} + +func (t *tunBroker) Ready() bool { + return true +} + +func (t *tunBroker) Health() bool { + return true +} + func (t *tunBroker) Init(opts ...broker.Option) error { for _, o := range opts { o(&t.opts) diff --git a/register/register.go b/register/register.go index 5f2bdab3..a5713cc4 100644 --- a/register/register.go +++ b/register/register.go @@ -29,17 +29,32 @@ var ( // and an abstraction over varying implementations // {consul, etcd, zookeeper, ...} type Register interface { + // Name returns register name Name() string + // Init initialize register Init(...Option) error + // Options returns options for register Options() Options + // Connect initialize connect to register Connect(context.Context) error + // Disconnect initialize discconection from register Disconnect(context.Context) error + // Register service in registry Register(context.Context, *Service, ...RegisterOption) error + // Deregister service from registry Deregister(context.Context, *Service, ...DeregisterOption) error + // LookupService in registry LookupService(context.Context, string, ...LookupOption) ([]*Service, error) + // ListServices in registry ListServices(context.Context, ...ListOption) ([]*Service, error) + // Watch registry events Watch(context.Context, ...WatchOption) (Watcher, error) + // String returns registry string representation String() string + // Live returns register liveness + // Live() bool + // Ready returns register readiness + // Ready() bool } // Service holds service register info diff --git a/server/noop.go b/server/noop.go index c4acf493..86fcfeb9 100644 --- a/server/noop.go +++ b/server/noop.go @@ -121,6 +121,18 @@ func (n *noopServer) newCodec(contentType string) (codec.Codec, error) { return nil, codec.ErrUnknownContentType } +func (n *noopServer) Live() bool { + return true +} + +func (n *noopServer) Ready() bool { + return true +} + +func (n *noopServer) Health() bool { + return true +} + func (n *noopServer) Handle(handler Handler) error { n.h = handler return nil diff --git a/server/options.go b/server/options.go index ffced1a4..4670cf63 100644 --- a/server/options.go +++ b/server/options.go @@ -12,7 +12,6 @@ import ( "go.unistack.org/micro/v3/logger" "go.unistack.org/micro/v3/metadata" "go.unistack.org/micro/v3/meter" - "go.unistack.org/micro/v3/network/transport" "go.unistack.org/micro/v3/options" "go.unistack.org/micro/v3/register" msync "go.unistack.org/micro/v3/sync" @@ -37,8 +36,6 @@ type Options struct { Logger logger.Logger // Meter holds the meter Meter meter.Meter - // Transport holds the transport - Transport transport.Transport /* // Router for requests @@ -100,7 +97,6 @@ func NewOptions(opts ...Option) Options { Tracer: tracer.DefaultTracer, Broker: broker.DefaultBroker, Register: register.DefaultRegister, - Transport: transport.DefaultTransport, Address: DefaultAddress, Name: DefaultName, Version: DefaultVersion, @@ -209,13 +205,6 @@ func Tracer(t tracer.Tracer) Option { } } -// Transport mechanism for communication e.g http, rabbitmq, etc -func Transport(t transport.Transport) Option { - return func(o *Options) { - o.Transport = t - } -} - // Metadata associated with the server func Metadata(md metadata.Metadata) Option { return func(o *Options) { @@ -249,14 +238,6 @@ func TLSConfig(t *tls.Config) Option { return func(o *Options) { // set the internal tls o.TLSConfig = t - - // set the default transport if one is not - // already set. Required for Init call below. - - // set the transport tls - _ = o.Transport.Init( - transport.TLSConfig(t), - ) } } diff --git a/server/server.go b/server/server.go index 49461e3a..8f9e1579 100644 --- a/server/server.go +++ b/server/server.go @@ -62,6 +62,12 @@ type Server interface { Stop() error // Server implementation String() string + // Live returns server liveness + Live() bool + // Ready returns server readiness + Ready() bool + // Health returns server health + Health() bool } type ( diff --git a/service.go b/service.go index cc827492..295e8c82 100644 --- a/service.go +++ b/service.go @@ -1,5 +1,5 @@ // Package micro is a pluggable framework for microservices -package micro // import "go.unistack.org/micro/v3" +package micro import ( "fmt" @@ -72,8 +72,14 @@ type Service interface { Start() error // Stop the service Stop() error - // The service implementation + // String service representation String() string + // Live returns service liveness + Live() bool + // Ready returns service readiness + Ready() bool + // Health returns service health + Health() bool } // RegisterHandler is syntactic sugar for registering a handler @@ -101,9 +107,7 @@ func (s *service) Name() string { return s.opts.Name } -// Init initialises options. Additionally it calls cmd.Init -// which parses command line flags. cmd.Init is only called -// on first Init. +// Init initialises options. // //nolint:gocyclo func (s *service) Init(opts ...Option) error { @@ -252,6 +256,63 @@ func (s *service) String() string { return s.opts.Name } +func (s *service) Live() bool { + for _, v := range s.opts.Brokers { + if !v.Live() { + return false + } + } + for _, v := range s.opts.Servers { + if !v.Live() { + return false + } + } + for _, v := range s.opts.Stores { + if !v.Live() { + return false + } + } + return true +} + +func (s *service) Ready() bool { + for _, v := range s.opts.Brokers { + if !v.Ready() { + return false + } + } + for _, v := range s.opts.Servers { + if !v.Ready() { + return false + } + } + for _, v := range s.opts.Stores { + if !v.Ready() { + return false + } + } + return true +} + +func (s *service) Health() bool { + for _, v := range s.opts.Brokers { + if !v.Health() { + return false + } + } + for _, v := range s.opts.Servers { + if !v.Health() { + return false + } + } + for _, v := range s.opts.Stores { + if !v.Health() { + return false + } + } + return true +} + //nolint:gocyclo func (s *service) Start() error { var err error @@ -281,10 +342,6 @@ func (s *service) Start() error { config.Loggers[0].Info(s.opts.Context, fmt.Sprintf("starting [service] %s version %s", s.Options().Name, s.Options().Version)) } - if len(s.opts.Servers) == 0 { - return fmt.Errorf("cant start nil server") - } - for _, reg := range s.opts.Registers { if err = reg.Connect(s.opts.Context); err != nil { return err diff --git a/store/memory/memory.go b/store/memory/memory.go index 11b41046..c0d5193a 100644 --- a/store/memory/memory.go +++ b/store/memory/memory.go @@ -149,6 +149,18 @@ func (m *memoryStore) Name() string { return m.opts.Name } +func (m *memoryStore) Live() bool { + return true +} + +func (m *memoryStore) Ready() bool { + return true +} + +func (m *memoryStore) Health() bool { + return true +} + func (m *memoryStore) Exists(ctx context.Context, key string, opts ...store.ExistsOption) error { if m.opts.LazyConnect { if err := m.connect(ctx); err != nil { @@ -279,3 +291,16 @@ func (m *memoryStore) connect(ctx context.Context) error { m.isConnected.CompareAndSwap(0, 1) return nil } + +func (m *memoryStore) Watch(ctx context.Context, opts ...store.WatchOption) (store.Watcher, error) { + return &watcher{}, nil +} + +type watcher struct{} + +func (w *watcher) Next() (store.Event, error) { + return nil, nil +} + +func (w *watcher) Stop() { +} diff --git a/store/noop.go b/store/noop.go index 83524901..24edafdd 100644 --- a/store/noop.go +++ b/store/noop.go @@ -23,6 +23,18 @@ type noopStore struct { isConnected atomic.Int32 } +func (n *noopStore) Live() bool { + return true +} + +func (n *noopStore) Ready() bool { + return true +} + +func (n *noopStore) Health() bool { + return true +} + func NewStore(opts ...Option) *noopStore { options := NewOptions(opts...) return &noopStore{opts: options} diff --git a/store/store.go b/store/store.go index 8235be5d..c5b8ac3e 100644 --- a/store/store.go +++ b/store/store.go @@ -45,7 +45,14 @@ type Store interface { Disconnect(ctx context.Context) error // String returns the name of the implementation. String() string + // Watch returns events watcher Watch(ctx context.Context, opts ...WatchOption) (Watcher, error) + // Live returns store liveness + Live() bool + // Ready returns store readiness + Ready() bool + // Health returns store health + Health() bool } type ( diff --git a/store/wrapper.go b/store/wrapper.go index a9221bae..7b62db40 100644 --- a/store/wrapper.go +++ b/store/wrapper.go @@ -70,3 +70,15 @@ func (w *NamespaceStore) String() string { func (w *NamespaceStore) Watch(ctx context.Context, opts ...WatchOption) (Watcher, error) { return w.s.Watch(ctx, opts...) } + +func (w *NamespaceStore) Live() bool { + return w.s.Live() +} + +func (w *NamespaceStore) Ready() bool { + return w.s.Ready() +} + +func (w *NamespaceStore) Health() bool { + return w.s.Health() +} diff --git a/util/io/io.go b/util/io/io.go deleted file mode 100644 index ed4d8fc0..00000000 --- a/util/io/io.go +++ /dev/null @@ -1,40 +0,0 @@ -// Package io is for io management -package io - -import ( - "io" - - "go.unistack.org/micro/v3/network/transport" -) - -type rwc struct { - socket transport.Socket -} - -func (r *rwc) Read(p []byte) (n int, err error) { - m := new(transport.Message) - if err := r.socket.Recv(m); err != nil { - return 0, err - } - copy(p, m.Body) - return len(m.Body), nil -} - -func (r *rwc) Write(p []byte) (n int, err error) { - err = r.socket.Send(&transport.Message{ - Body: p, - }) - if err != nil { - return 0, err - } - return len(p), nil -} - -func (r *rwc) Close() error { - return r.socket.Close() -} - -// NewRWC returns a new ReadWriteCloser -func NewRWC(sock transport.Socket) io.ReadWriteCloser { - return &rwc{sock} -} diff --git a/util/pool/default.go b/util/pool/default.go deleted file mode 100644 index 7f61c281..00000000 --- a/util/pool/default.go +++ /dev/null @@ -1,118 +0,0 @@ -package pool - -import ( - "context" - "sync" - "time" - - "go.unistack.org/micro/v3/network/transport" - "go.unistack.org/micro/v3/util/id" -) - -type pool struct { - tr transport.Transport - conns map[string][]*poolConn - size int - ttl time.Duration - sync.Mutex -} - -type poolConn struct { - created time.Time - transport.Client - id string -} - -func newPool(options Options) *pool { - return &pool{ - size: options.Size, - tr: options.Transport, - ttl: options.TTL, - conns: make(map[string][]*poolConn), - } -} - -func (p *pool) Close() error { - p.Lock() - for k, c := range p.conns { - for _, conn := range c { - conn.Client.Close() - } - delete(p.conns, k) - } - p.Unlock() - return nil -} - -// NoOp the Close since we manage it -func (p *poolConn) Close() error { - return nil -} - -func (p *poolConn) ID() string { - return p.id -} - -func (p *poolConn) Created() time.Time { - return p.created -} - -func (p *pool) Get(ctx context.Context, addr string, opts ...transport.DialOption) (Conn, error) { - p.Lock() - conns := p.conns[addr] - - // while we have conns check age and then return one - // otherwise we'll create a new conn - for len(conns) > 0 { - conn := conns[len(conns)-1] - conns = conns[:len(conns)-1] - p.conns[addr] = conns - - // if conn is old kill it and move on - if d := time.Since(conn.Created()); d > p.ttl { - conn.Client.Close() - continue - } - - // we got a good conn, lets unlock and return it - p.Unlock() - - return conn, nil - } - - p.Unlock() - - // create new conn - c, err := p.tr.Dial(ctx, addr, opts...) - if err != nil { - return nil, err - } - id, err := id.New() - if err != nil { - return nil, err - } - return &poolConn{ - Client: c, - id: id, - created: time.Now(), - }, nil -} - -func (p *pool) Release(conn Conn, err error) error { - // don't store the conn if it has errored - if err != nil { - return conn.(*poolConn).Client.Close() - } - - // otherwise put it back for reuse - p.Lock() - conns := p.conns[conn.Remote()] - if len(conns) >= p.size { - p.Unlock() - return conn.(*poolConn).Client.Close() - } - p.conns[conn.Remote()] = append(conns, conn.(*poolConn)) - p.Unlock() - - return nil -} diff --git a/util/pool/default_test.go b/util/pool/default_test.go deleted file mode 100644 index 7fab0f6b..00000000 --- a/util/pool/default_test.go +++ /dev/null @@ -1,92 +0,0 @@ -//go:build ignore -// +build ignore - -package pool - -import ( - "testing" - "time" - - "go.unistack.org/micro/v3/network/transport" - "go.unistack.org/micro/v3/network/transport/memory" -) - -func testPool(t *testing.T, size int, ttl time.Duration) { - // mock transport - tr := memory.NewTransport() - - options := Options{ - TTL: ttl, - Size: size, - Transport: tr, - } - // zero pool - p := newPool(options) - - // listen - l, err := tr.Listen(":0") - if err != nil { - t.Fatal(err) - } - defer l.Close() - - // accept loop - go func() { - for { - if err := l.Accept(func(s transport.Socket) { - for { - var msg transport.Message - if err := s.Recv(&msg); err != nil { - return - } - if err := s.Send(&msg); err != nil { - return - } - } - }); err != nil { - return - } - } - }() - - for i := 0; i < 10; i++ { - // get a conn - c, err := p.Get(l.Addr()) - if err != nil { - t.Fatal(err) - } - - msg := &transport.Message{ - Body: []byte(`hello world`), - } - - if err := c.Send(msg); err != nil { - t.Fatal(err) - } - - var rcv transport.Message - - if err := c.Recv(&rcv); err != nil { - t.Fatal(err) - } - - if string(rcv.Body) != string(msg.Body) { - t.Fatalf("got %v, expected %v", rcv.Body, msg.Body) - } - - // release the conn - p.Release(c, nil) - - p.Lock() - if i := len(p.conns[l.Addr()]); i > size { - p.Unlock() - t.Fatalf("pool size %d is greater than expected %d", i, size) - } - p.Unlock() - } -} - -func TestClientPool(t *testing.T) { - testPool(t, 0, time.Minute) - testPool(t, 2, time.Minute) -} diff --git a/util/pool/options.go b/util/pool/options.go deleted file mode 100644 index 3edfc310..00000000 --- a/util/pool/options.go +++ /dev/null @@ -1,38 +0,0 @@ -package pool - -import ( - "time" - - "go.unistack.org/micro/v3/network/transport" -) - -// Options struct -type Options struct { - Transport transport.Transport - TTL time.Duration - Size int -} - -// Option func signature -type Option func(*Options) - -// Size sets the size -func Size(i int) Option { - return func(o *Options) { - o.Size = i - } -} - -// Transport sets the transport -func Transport(t transport.Transport) Option { - return func(o *Options) { - o.Transport = t - } -} - -// TTL specifies ttl -func TTL(t time.Duration) Option { - return func(o *Options) { - o.TTL = t - } -} diff --git a/util/pool/pool.go b/util/pool/pool.go deleted file mode 100644 index 2433cd59..00000000 --- a/util/pool/pool.go +++ /dev/null @@ -1,38 +0,0 @@ -// Package pool is a connection pool -package pool - -import ( - "context" - "time" - - "go.unistack.org/micro/v3/network/transport" -) - -// Pool is an interface for connection pooling -type Pool interface { - // Close the pool - Close() error - // Get a connection - Get(ctx context.Context, addr string, opts ...transport.DialOption) (Conn, error) - // Release the connection - Release(c Conn, status error) error -} - -// Conn conn pool interface -type Conn interface { - // unique id of connection - ID() string - // time it was created - Created() time.Time - // embedded connection - transport.Client -} - -// NewPool creates new connection pool -func NewPool(opts ...Option) Pool { - options := Options{} - for _, o := range opts { - o(&options) - } - return newPool(options) -} diff --git a/util/time/duration_test.go b/util/time/duration_test.go index 0324e68a..af5f1434 100644 --- a/util/time/duration_test.go +++ b/util/time/duration_test.go @@ -23,7 +23,7 @@ func TestMarshalYAML(t *testing.T) { func TestUnmarshalYAML(t *testing.T) { type str struct { - TTL Duration `yaml:"ttl"` + TTL *Duration `yaml:"ttl"` } v := &str{} var err error @@ -31,14 +31,14 @@ func TestUnmarshalYAML(t *testing.T) { err = yaml.Unmarshal([]byte(`{"ttl":"10ms"}`), v) if err != nil { t.Fatal(err) - } else if v.TTL != 10000000 { + } else if *(v.TTL) != 10000000 { t.Fatalf("invalid duration %v != 10000000", v.TTL) } err = yaml.Unmarshal([]byte(`{"ttl":"1y"}`), v) if err != nil { t.Fatal(err) - } else if v.TTL != 31622400000000000 { + } else if *(v.TTL) != 31622400000000000 { t.Fatalf("invalid duration %v != 31622400000000000", v.TTL) } } From bf4143cde574f27d69db9eb685a7166c915f1eb3 Mon Sep 17 00:00:00 2001 From: Vasiliy Tolstov Date: Tue, 3 Dec 2024 01:11:08 +0300 Subject: [PATCH 59/59] replace default go resolver with caching resolver Signed-off-by: Vasiliy Tolstov --- resolver/dns/dns.go | 10 +- service.go | 5 + util/dns/cache.go | 377 +++++++++++++++++++++++++++++++++++++++++ util/dns/cache_test.go | 16 ++ util/dns/conn.go | 178 +++++++++++++++++++ 5 files changed, 581 insertions(+), 5 deletions(-) create mode 100644 util/dns/cache.go create mode 100644 util/dns/cache_test.go create mode 100644 util/dns/conn.go diff --git a/resolver/dns/dns.go b/resolver/dns/dns.go index e80a9c8b..35c80c67 100644 --- a/resolver/dns/dns.go +++ b/resolver/dns/dns.go @@ -12,9 +12,9 @@ import ( // Resolver is a DNS network resolve type Resolver struct { - sync.RWMutex goresolver *net.Resolver Address string + mu sync.RWMutex } // Resolve tries to resolve endpoint address @@ -39,12 +39,12 @@ func (r *Resolver) Resolve(name string) ([]*resolver.Record, error) { return []*resolver.Record{rec}, nil } - r.RLock() + r.mu.RLock() goresolver := r.goresolver - r.RUnlock() + r.mu.RUnlock() if goresolver == nil { - r.Lock() + r.mu.Lock() r.goresolver = &net.Resolver{ Dial: func(ctx context.Context, _ string, _ string) (net.Conn, error) { d := net.Dialer{ @@ -53,7 +53,7 @@ func (r *Resolver) Resolve(name string) ([]*resolver.Record, error) { return d.DialContext(ctx, "udp", r.Address) }, } - r.Unlock() + r.mu.Unlock() } addrs, err := goresolver.LookupIP(context.TODO(), "ip", host) diff --git a/service.go b/service.go index 295e8c82..3c782035 100644 --- a/service.go +++ b/service.go @@ -3,7 +3,9 @@ package micro import ( "fmt" + "net" "sync" + "time" "github.com/KimMachineGun/automemlimit/memlimit" "go.uber.org/automaxprocs/maxprocs" @@ -17,6 +19,7 @@ import ( "go.unistack.org/micro/v3/server" "go.unistack.org/micro/v3/store" "go.unistack.org/micro/v3/tracer" + utildns "go.unistack.org/micro/v3/util/dns" ) func init() { @@ -30,6 +33,8 @@ func init() { ), ), ) + + net.DefaultResolver = utildns.NewNetResolver(utildns.Timeout(1 * time.Second)) } // Service is an interface that wraps the lower level components. diff --git a/util/dns/cache.go b/util/dns/cache.go new file mode 100644 index 00000000..051cc041 --- /dev/null +++ b/util/dns/cache.go @@ -0,0 +1,377 @@ +package dns + +import ( + "context" + "math" + "net" + "sync" + "time" +) + +// DialFunc is a [net.Resolver.Dial] function. +type DialFunc func(ctx context.Context, network, address string) (net.Conn, error) + +// NewNetResolver creates a caching [net.Resolver] that uses parent to resolve names. +func NewNetResolver(opts ...Option) *net.Resolver { + options := Options{Resolver: &net.Resolver{}} + + for _, o := range opts { + o(&options) + } + + return &net.Resolver{ + PreferGo: true, + StrictErrors: options.Resolver.StrictErrors, + Dial: NewNetDialer(options.Resolver.Dial, append(opts, Resolver(options.Resolver))...), + } +} + +// NewNetDialer adds caching to a [net.Resolver.Dial] function. +func NewNetDialer(parent DialFunc, opts ...Option) DialFunc { + cache := cache{dial: parent, opts: Options{}} + for _, o := range opts { + o(&cache.opts) + } + if cache.opts.MaxCacheEntries == 0 { + cache.opts.MaxCacheEntries = DefaultMaxCacheEntries + } + return func(ctx context.Context, network, address string) (net.Conn, error) { + conn := &dnsConn{} + conn.roundTrip = cachingRoundTrip(&cache, network, address) + return conn, nil + } +} + +const DefaultMaxCacheEntries = 300 + +// A Option customizes the resolver cache. +type Option func(*Options) + +type Options struct { + Resolver *net.Resolver + MaxCacheEntries int + MaxCacheTTL time.Duration + MinCacheTTL time.Duration + NegativeCache bool + PreferIPV4 bool + PreferIPV6 bool + Timeout time.Duration +} + +// MaxCacheEntries sets the maximum number of entries to cache. +// If zero, [DefaultMaxCacheEntries] is used; negative means no limit. +func MaxCacheEntries(n int) Option { + return func(o *Options) { + o.MaxCacheEntries = n + } +} + +// MaxCacheTTL sets the maximum time-to-live for entries in the cache. +func MaxCacheTTL(td time.Duration) Option { + return func(o *Options) { + o.MaxCacheTTL = td + } +} + +// MinCacheTTL sets the minimum time-to-live for entries in the cache. +func MinCacheTTL(td time.Duration) Option { + return func(o *Options) { + o.MinCacheTTL = td + } +} + +// NegativeCache sets whether to cache negative responses. +func NegativeCache(b bool) Option { + return func(o *Options) { + o.NegativeCache = b + } +} + +// Timeout sets upstream *net.Resolver timeout +func Timeout(td time.Duration) Option { + return func(o *Options) { + o.Timeout = td + } +} + +// Resolver sets upstream *net.Resolver. +func Resolver(r *net.Resolver) Option { + return func(o *Options) { + o.Resolver = r + } +} + +// PreferIPV4 resolve ipv4 records. +func PreferIPV4(b bool) Option { + return func(o *Options) { + o.PreferIPV4 = b + } +} + +// PreferIPV6 resolve ipv4 records. +func PreferIPV6(b bool) Option { + return func(o *Options) { + o.PreferIPV6 = b + } +} + +type cache struct { + sync.RWMutex + + dial DialFunc + entries map[string]cacheEntry + + opts Options +} + +type cacheEntry struct { + deadline time.Time + value string +} + +func (c *cache) put(req string, res string) { + // ignore uncacheable/unparseable answers + if invalid(req, res) { + return + } + + // ignore errors (if requested) + if nameError(res) && !c.opts.NegativeCache { + return + } + + // ignore uncacheable/unparseable answers + ttl := getTTL(res) + if ttl <= 0 { + return + } + + // adjust TTL + if ttl < c.opts.MinCacheTTL { + ttl = c.opts.MinCacheTTL + } + // maxTTL overrides minTTL + if ttl > c.opts.MaxCacheTTL && c.opts.MaxCacheTTL != 0 { + ttl = c.opts.MaxCacheTTL + } + + c.Lock() + defer c.Unlock() + if c.entries == nil { + c.entries = make(map[string]cacheEntry) + } + + // do some cache evition + var tested, evicted int + for k, e := range c.entries { + if time.Until(e.deadline) <= 0 { + // delete expired entry + delete(c.entries, k) + evicted++ + } + tested++ + + if tested < 8 { + continue + } + if evicted == 0 && c.opts.MaxCacheEntries > 0 && len(c.entries) >= c.opts.MaxCacheEntries { + // delete at least one entry + delete(c.entries, k) + } + break + } + + // remove message IDs + c.entries[req[2:]] = cacheEntry{ + deadline: time.Now().Add(ttl), + value: res[2:], + } +} + +func (c *cache) get(req string) (res string) { + // ignore invalid messages + if len(req) < 12 { + return "" + } + if req[2] >= 0x7f { + return "" + } + + c.RLock() + defer c.RUnlock() + + if c.entries == nil { + return "" + } + + // remove message ID + entry, ok := c.entries[req[2:]] + if ok && time.Until(entry.deadline) > 0 { + // prepend correct ID + return req[:2] + entry.value + } + return "" +} + +func invalid(req string, res string) bool { + if len(req) < 12 || len(res) < 12 { // header size + return true + } + if req[0] != res[0] || req[1] != res[1] { // IDs match + return true + } + if req[2] >= 0x7f || res[2] < 0x7f { // query, response + return true + } + if req[2]&0x7a != 0 || res[2]&0x7a != 0 { // standard query, not truncated + return true + } + if res[3]&0xf != 0 && res[3]&0xf != 3 { // no error, or name error + return true + } + return false +} + +func nameError(res string) bool { + return res[3]&0xf == 3 +} + +func getTTL(msg string) time.Duration { + ttl := math.MaxInt32 + + qdcount := getUint16(msg[4:]) + ancount := getUint16(msg[6:]) + nscount := getUint16(msg[8:]) + arcount := getUint16(msg[10:]) + rdcount := ancount + nscount + arcount + + msg = msg[12:] // skip header + + // skip questions + for i := 0; i < qdcount; i++ { + name := getNameLen(msg) + if name < 0 || name+4 > len(msg) { + return -1 + } + msg = msg[name+4:] + } + + // parse records + for i := 0; i < rdcount; i++ { + name := getNameLen(msg) + if name < 0 || name+10 > len(msg) { + return -1 + } + rtyp := getUint16(msg[name+0:]) + rttl := getUint32(msg[name+4:]) + rlen := getUint16(msg[name+8:]) + if name+10+rlen > len(msg) { + return -1 + } + // skip EDNS OPT since it doesn't have a TTL + if rtyp != 41 && rttl < ttl { + ttl = rttl + } + msg = msg[name+10+rlen:] + } + + return time.Duration(ttl) * time.Second +} + +func getNameLen(msg string) int { + i := 0 + for i < len(msg) { + if msg[i] == 0 { + // end of name + i += 1 + break + } + if msg[i] >= 0xc0 { + // compressed name + i += 2 + break + } + if msg[i] >= 0x40 { + // reserved + return -1 + } + i += int(msg[i] + 1) + } + return i +} + +func getUint16(s string) int { + return int(s[1]) | int(s[0])<<8 +} + +func getUint32(s string) int { + return int(s[3]) | int(s[2])<<8 | int(s[1])<<16 | int(s[0])<<24 +} + +func cachingRoundTrip(cache *cache, network, address string) roundTripper { + return func(ctx context.Context, req string) (res string, err error) { + // check cache + if res := cache.get(req); res != "" { + return res, nil + } + + switch { + case cache.opts.PreferIPV4 && cache.opts.PreferIPV6: + network = "udp" + case cache.opts.PreferIPV4: + network = "udp4" + case cache.opts.PreferIPV6: + network = "udp6" + default: + network = "udp" + } + + if cache.opts.Timeout > 0 { + var cancel func() + ctx, cancel = context.WithTimeout(ctx, cache.opts.Timeout) + defer cancel() + } + + // dial connection + var conn net.Conn + if cache.dial != nil { + conn, err = cache.dial(ctx, network, address) + } else { + var d net.Dialer + conn, err = d.DialContext(ctx, network, address) + } + if err != nil { + return "", err + } + + ctx, cancel := context.WithCancel(ctx) + go func() { + <-ctx.Done() + conn.Close() + }() + defer cancel() + + if t, ok := ctx.Deadline(); ok { + err = conn.SetDeadline(t) + if err != nil { + return "", err + } + } + + // send request + err = writeMessage(conn, req) + if err != nil { + return "", err + } + + // read response + res, err = readMessage(conn) + if err != nil { + return "", err + } + + // cache response + cache.put(req, res) + return res, nil + } +} diff --git a/util/dns/cache_test.go b/util/dns/cache_test.go new file mode 100644 index 00000000..6e2fb1d1 --- /dev/null +++ b/util/dns/cache_test.go @@ -0,0 +1,16 @@ +package dns + +import ( + "net" + "testing" +) + +func TestCache(t *testing.T) { + net.DefaultResolver = NewNetResolver(PreferIPV4(true)) + + addrs, err := net.LookupHost("unistack.org") + if err != nil { + t.Fatal(err) + } + t.Logf("addrs %v", addrs) +} diff --git a/util/dns/conn.go b/util/dns/conn.go new file mode 100644 index 00000000..f6057a96 --- /dev/null +++ b/util/dns/conn.go @@ -0,0 +1,178 @@ +package dns + +import ( + "bytes" + "context" + "io" + "net" + "strings" + "sync" + "time" +) + +type dnsConn struct { + sync.Mutex + + ibuf bytes.Buffer + obuf bytes.Buffer + + ctx context.Context + cancel context.CancelFunc + deadline time.Time + roundTrip roundTripper +} + +type roundTripper func(ctx context.Context, req string) (res string, err error) + +func (c *dnsConn) Read(b []byte) (n int, err error) { + imsg, n, err := c.drainBuffers(b) + if n != 0 || err != nil { + return n, err + } + + ctx, cancel := c.childContext() + omsg, err := c.roundTrip(ctx, imsg) + cancel() + if err != nil { + return 0, err + } + + return c.fillBuffer(b, omsg) +} + +func (c *dnsConn) Write(b []byte) (n int, err error) { + c.Lock() + defer c.Unlock() + return c.ibuf.Write(b) +} + +func (c *dnsConn) Close() error { + c.Lock() + cancel := c.cancel + c.Unlock() + + if cancel != nil { + cancel() + } + return nil +} + +func (c *dnsConn) LocalAddr() net.Addr { + return nil +} + +func (c *dnsConn) RemoteAddr() net.Addr { + return nil +} + +func (c *dnsConn) SetDeadline(t time.Time) error { + c.SetReadDeadline(t) + c.SetWriteDeadline(t) + return nil +} + +func (c *dnsConn) SetReadDeadline(t time.Time) error { + c.Lock() + defer c.Unlock() + c.deadline = t + return nil +} + +func (c *dnsConn) SetWriteDeadline(t time.Time) error { + // writes do not timeout + return nil +} + +func (c *dnsConn) drainBuffers(b []byte) (string, int, error) { + c.Lock() + defer c.Unlock() + + // drain the output buffer + if c.obuf.Len() > 0 { + n, err := c.obuf.Read(b) + return "", n, err + } + + // otherwise, get the next message from the input buffer + sz := c.ibuf.Next(2) + if len(sz) < 2 { + return "", 0, io.ErrUnexpectedEOF + } + + size := int64(sz[0])<<8 | int64(sz[1]) + + var str strings.Builder + _, err := io.CopyN(&str, &c.ibuf, size) + if err == io.EOF { + return "", 0, io.ErrUnexpectedEOF + } + if err != nil { + return "", 0, err + } + return str.String(), 0, nil +} + +func (c *dnsConn) fillBuffer(b []byte, str string) (int, error) { + c.Lock() + defer c.Unlock() + c.obuf.WriteByte(byte(len(str) >> 8)) + c.obuf.WriteByte(byte(len(str))) + c.obuf.WriteString(str) + return c.obuf.Read(b) +} + +func (c *dnsConn) childContext() (context.Context, context.CancelFunc) { + c.Lock() + defer c.Unlock() + if c.ctx == nil { + c.ctx, c.cancel = context.WithCancel(context.Background()) + } + return context.WithDeadline(c.ctx, c.deadline) +} + +func writeMessage(conn net.Conn, msg string) error { + var buf []byte + if _, ok := conn.(net.PacketConn); ok { + buf = []byte(msg) + } else { + buf = make([]byte, len(msg)+2) + buf[0] = byte(len(msg) >> 8) + buf[1] = byte(len(msg)) + copy(buf[2:], msg) + } + // SHOULD do a single write on TCP (RFC 7766, section 8). + // MUST do a single write on UDP. + _, err := conn.Write(buf) + return err +} + +func readMessage(c net.Conn) (string, error) { + if _, ok := c.(net.PacketConn); ok { + // RFC 1035 specifies 512 as the maximum message size for DNS over UDP. + // RFC 6891 OTOH suggests 4096 as the maximum payload size for EDNS. + b := make([]byte, 4096) + n, err := c.Read(b) + if err != nil { + return "", err + } + return string(b[:n]), nil + } else { + var sz [2]byte + _, err := io.ReadFull(c, sz[:]) + if err != nil { + return "", err + } + + size := int64(sz[0])<<8 | int64(sz[1]) + + var str strings.Builder + _, err = io.CopyN(&str, c, size) + if err == io.EOF { + return "", io.ErrUnexpectedEOF + } + if err != nil { + return "", err + } + return str.String(), nil + } +}