Compare commits

...

5 Commits

Author SHA1 Message Date
969e459e3d add tracer enabled status
Some checks failed
test / test (push) Failing after 17m28s
coverage / build (push) Failing after 17m37s
sync / sync (push) Has been cancelled
Signed-off-by: Vasiliy Tolstov <v.tolstov@unistack.org>
2025-05-19 09:34:24 +03:00
vtolstov
b2cd7d8b8e Apply Code Coverage Badge 2025-05-18 16:02:55 +00:00
6d85d3ee41 disable message pool by default
Some checks failed
sync / sync (push) Has been skipped
test / test (push) Successful in 4m47s
coverage / build (push) Failing after 18m8s
Signed-off-by: Vasiliy Tolstov <v.tolstov@unistack.org>
2025-05-18 18:58:57 +03:00
d0978fb314 Обновить README.md
Some checks failed
coverage / build (push) Failing after 18m14s
test / test (push) Successful in 3m43s
sync / sync (push) Has been skipped
2025-05-12 11:05:30 +03:00
vtolstov
0bd81d1253 Apply Code Coverage Badge 2025-05-10 22:53:42 +00:00
7 changed files with 98 additions and 65 deletions

View File

@@ -1,9 +1,2 @@
# broker-kgo # micro-broker-kgo
![Coverage](https://img.shields.io/badge/Coverage-65.8%25-yellow)
TODO:
1) экспортируем текущий оффсет для каждой партиции в группе
2) экспортируем лаг для группы
3) мониторим
1) если есть лаг больше нуля
2) если дельта оффсета за нужное нам время не

3
go.mod
View File

@@ -8,8 +8,7 @@ require (
github.com/twmb/franz-go/pkg/kfake v0.0.0-20250508175730-72e1646135e3 github.com/twmb/franz-go/pkg/kfake v0.0.0-20250508175730-72e1646135e3
github.com/twmb/franz-go/pkg/kmsg v1.11.2 github.com/twmb/franz-go/pkg/kmsg v1.11.2
go.opentelemetry.io/otel v1.35.0 go.opentelemetry.io/otel v1.35.0
go.unistack.org/micro-codec-json/v4 v4.1.0 go.unistack.org/micro/v4 v4.1.14
go.unistack.org/micro/v4 v4.1.13
) )
require ( require (

8
go.sum
View File

@@ -24,8 +24,6 @@ github.com/spf13/cast v1.8.0 h1:gEN9K4b8Xws4EX0+a0reLmhq8moKn7ntRlQYgjPeCDk=
github.com/spf13/cast v1.8.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo= github.com/spf13/cast v1.8.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo=
github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/twmb/franz-go v1.19.0 h1:FzBAPUeaip68X9cbLDesgQesa5zxKVaZMk+du98vj3c=
github.com/twmb/franz-go v1.19.0/go.mod h1:4kFJ5tmbbl7asgwAGVuyG1ZMx0NNpYk7EqflvWfPCpM=
github.com/twmb/franz-go v1.19.1 h1:cOhDFUkGvUFHSQ7UYW6bO77BJa2fYEk5mA2AX+1NIdE= github.com/twmb/franz-go v1.19.1 h1:cOhDFUkGvUFHSQ7UYW6bO77BJa2fYEk5mA2AX+1NIdE=
github.com/twmb/franz-go v1.19.1/go.mod h1:4kFJ5tmbbl7asgwAGVuyG1ZMx0NNpYk7EqflvWfPCpM= github.com/twmb/franz-go v1.19.1/go.mod h1:4kFJ5tmbbl7asgwAGVuyG1ZMx0NNpYk7EqflvWfPCpM=
github.com/twmb/franz-go/pkg/kadm v1.16.0 h1:STMs1t5lYR5mR974PSiwNzE5TvsosByTp+rKXLOhAjE= github.com/twmb/franz-go/pkg/kadm v1.16.0 h1:STMs1t5lYR5mR974PSiwNzE5TvsosByTp+rKXLOhAjE=
@@ -36,12 +34,10 @@ github.com/twmb/franz-go/pkg/kmsg v1.11.2 h1:hIw75FpwcAjgeyfIGFqivAvwC5uNIOWRGvQ
github.com/twmb/franz-go/pkg/kmsg v1.11.2/go.mod h1:CFfkkLysDNmukPYhGzuUcDtf46gQSqCZHMW1T4Z+wDE= github.com/twmb/franz-go/pkg/kmsg v1.11.2/go.mod h1:CFfkkLysDNmukPYhGzuUcDtf46gQSqCZHMW1T4Z+wDE=
go.opentelemetry.io/otel v1.35.0 h1:xKWKPxrxB6OtMCbmMY021CqC45J+3Onta9MqjhnusiQ= go.opentelemetry.io/otel v1.35.0 h1:xKWKPxrxB6OtMCbmMY021CqC45J+3Onta9MqjhnusiQ=
go.opentelemetry.io/otel v1.35.0/go.mod h1:UEqy8Zp11hpkUrL73gSlELM0DupHoiq72dR+Zqel/+Y= go.opentelemetry.io/otel v1.35.0/go.mod h1:UEqy8Zp11hpkUrL73gSlELM0DupHoiq72dR+Zqel/+Y=
go.unistack.org/micro-codec-json/v4 v4.1.0 h1:iydeSkt3ee7IPU0dHHKlGN97lw+YFQasBk9rdv0woYA=
go.unistack.org/micro-codec-json/v4 v4.1.0/go.mod h1:aUg86elSlURSynTAetDAAXj/VzFDwwcg92QNrRzcvrM=
go.unistack.org/micro-proto/v4 v4.1.0 h1:qPwL2n/oqh9RE3RTTDgt28XK3QzV597VugQPaw9lKUk= go.unistack.org/micro-proto/v4 v4.1.0 h1:qPwL2n/oqh9RE3RTTDgt28XK3QzV597VugQPaw9lKUk=
go.unistack.org/micro-proto/v4 v4.1.0/go.mod h1:ArmK7o+uFvxSY3dbJhKBBX4Pm1rhWdLEFf3LxBrMtec= go.unistack.org/micro-proto/v4 v4.1.0/go.mod h1:ArmK7o+uFvxSY3dbJhKBBX4Pm1rhWdLEFf3LxBrMtec=
go.unistack.org/micro/v4 v4.1.13 h1:1IEQwiIwHdypZN4dWmsWN83Plq9bdkQ4U8aYZsT17s8= go.unistack.org/micro/v4 v4.1.14 h1:6EotPq9kz/gaFb5YulHdKuuUwmj/7Hk44DpOlzh/A6k=
go.unistack.org/micro/v4 v4.1.13/go.mod h1:xleO2M5Yxh4s6I+RUcLrEpUjobefh+71ctrdIfn7TUs= go.unistack.org/micro/v4 v4.1.14/go.mod h1:xleO2M5Yxh4s6I+RUcLrEpUjobefh+71ctrdIfn7TUs=
golang.org/x/crypto v0.38.0 h1:jt+WWG8IZlBnVbomuhg2Mdq0+BBQaHbtqHEFEigjUV8= golang.org/x/crypto v0.38.0 h1:jt+WWG8IZlBnVbomuhg2Mdq0+BBQaHbtqHEFEigjUV8=
golang.org/x/crypto v0.38.0/go.mod h1:MvrbAqul58NNYPKnOra203SB9vpuZW0e+RRZV+Ggqjw= golang.org/x/crypto v0.38.0/go.mod h1:MvrbAqul58NNYPKnOra203SB9vpuZW0e+RRZV+Ggqjw=
google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY=

5
kgo.go
View File

@@ -478,11 +478,15 @@ func (b *Broker) fnSubscribe(ctx context.Context, topic string, handler interfac
} }
} }
var messagePool bool
var fatalOnError bool var fatalOnError bool
if b.opts.Context != nil { if b.opts.Context != nil {
if v, ok := b.opts.Context.Value(fatalOnErrorKey{}).(bool); ok && v { if v, ok := b.opts.Context.Value(fatalOnErrorKey{}).(bool); ok && v {
fatalOnError = v fatalOnError = v
} }
if v, ok := b.opts.Context.Value(subscribeMessagePoolKey{}).(bool); ok && v {
messagePool = v
}
} }
if options.Context != nil { if options.Context != nil {
@@ -500,6 +504,7 @@ func (b *Broker) fnSubscribe(ctx context.Context, topic string, handler interfac
done: make(chan struct{}), done: make(chan struct{}),
fatalOnError: fatalOnError, fatalOnError: fatalOnError,
connected: b.connected, connected: b.connected,
messagePool: messagePool,
} }
kopts := append(b.kopts, kopts := append(b.kopts,

View File

@@ -109,3 +109,10 @@ type publishPromiseKey struct{}
func PublishPromise(fn func(*kgo.Record, error)) broker.PublishOption { func PublishPromise(fn func(*kgo.Record, error)) broker.PublishOption {
return broker.SetPublishOption(publishPromiseKey{}, fn) return broker.SetPublishOption(publishPromiseKey{}, fn)
} }
type subscribeMessagePoolKey struct{}
// SubscribeMessagePool optionaly enabled/disable message pool
func SubscribeMessagePool(b bool) broker.SubscribeOption {
return broker.SetSubscribeOption(subscribeMessagePoolKey{}, b)
}

View File

@@ -35,6 +35,7 @@ type consumer struct {
opts broker.SubscribeOptions opts broker.SubscribeOptions
handler interface{} handler interface{}
connected *atomic.Uint32 connected *atomic.Uint32
messagePool bool
} }
type Subscriber struct { type Subscriber struct {
@@ -42,7 +43,7 @@ type Subscriber struct {
c *kgo.Client c *kgo.Client
htracer *hookTracer htracer *hookTracer
topic string topic string
messagePool bool
handler interface{} handler interface{}
done chan struct{} done chan struct{}
kopts broker.Options kopts broker.Options
@@ -224,6 +225,7 @@ func (s *Subscriber) assigned(_ context.Context, c *kgo.Client, assigned map[str
done: make(chan struct{}), done: make(chan struct{}),
recs: make(chan kgo.FetchTopicPartition, 100), recs: make(chan kgo.FetchTopicPartition, 100),
handler: s.handler, handler: s.handler,
messagePool: s.messagePool,
kopts: s.kopts, kopts: s.kopts,
opts: s.opts, opts: s.opts,
connected: s.connected, connected: s.connected,
@@ -245,6 +247,8 @@ func (pc *consumer) consume() {
defer pc.kopts.Logger.Debug(pc.kopts.Context, fmt.Sprintf("killing, topic %s partition %d", pc.topic, pc.partition)) defer pc.kopts.Logger.Debug(pc.kopts.Context, fmt.Sprintf("killing, topic %s partition %d", pc.topic, pc.partition))
} }
var pm *kgoMessage
for { for {
select { select {
case <-pc.quit: case <-pc.quit:
@@ -254,50 +258,60 @@ func (pc *consumer) consume() {
ctx, sp := pc.htracer.WithProcessSpan(record) ctx, sp := pc.htracer.WithProcessSpan(record)
ts := time.Now() ts := time.Now()
pc.kopts.Meter.Counter(semconv.SubscribeMessageInflight, "endpoint", record.Topic, "topic", record.Topic).Inc() pc.kopts.Meter.Counter(semconv.SubscribeMessageInflight, "endpoint", record.Topic, "topic", record.Topic).Inc()
p := messagePool.Get().(*kgoMessage)
p.body = record.Value if pc.messagePool {
p.topic = record.Topic pm = messagePool.Get().(*kgoMessage)
p.ack = false } else {
p.hdr = metadata.New(len(record.Headers)) pm = &kgoMessage{}
p.ctx = ctx }
pm.body = record.Value
pm.topic = record.Topic
pm.ack = false
pm.hdr = metadata.New(len(record.Headers))
pm.ctx = ctx
for _, hdr := range record.Headers { for _, hdr := range record.Headers {
p.hdr.Set(hdr.Key, string(hdr.Value)) pm.hdr.Set(hdr.Key, string(hdr.Value))
} }
switch h := pc.handler.(type) { switch h := pc.handler.(type) {
case func(broker.Message) error: case func(broker.Message) error:
err = h(p) err = h(pm)
case func([]broker.Message) error: case func([]broker.Message) error:
err = h([]broker.Message{p}) err = h([]broker.Message{pm})
} }
pc.kopts.Meter.Counter(semconv.SubscribeMessageInflight, "endpoint", record.Topic, "topic", record.Topic).Dec() pc.kopts.Meter.Counter(semconv.SubscribeMessageInflight, "endpoint", record.Topic, "topic", record.Topic).Dec()
if err != nil { if err != nil {
if sp != nil {
sp.SetStatus(tracer.SpanStatusError, err.Error()) sp.SetStatus(tracer.SpanStatusError, err.Error())
}
pc.kopts.Meter.Counter(semconv.SubscribeMessageTotal, "endpoint", record.Topic, "topic", record.Topic, "status", "failure").Inc() pc.kopts.Meter.Counter(semconv.SubscribeMessageTotal, "endpoint", record.Topic, "topic", record.Topic, "status", "failure").Inc()
} else if pc.opts.AutoAck { } else if pc.opts.AutoAck {
p.ack = true pm.ack = true
} }
te := time.Since(ts) te := time.Since(ts)
pc.kopts.Meter.Summary(semconv.SubscribeMessageLatencyMicroseconds, "endpoint", record.Topic, "topic", record.Topic).Update(te.Seconds()) pc.kopts.Meter.Summary(semconv.SubscribeMessageLatencyMicroseconds, "endpoint", record.Topic, "topic", record.Topic).Update(te.Seconds())
pc.kopts.Meter.Histogram(semconv.SubscribeMessageDurationSeconds, "endpoint", record.Topic, "topic", record.Topic).Update(te.Seconds()) pc.kopts.Meter.Histogram(semconv.SubscribeMessageDurationSeconds, "endpoint", record.Topic, "topic", record.Topic).Update(te.Seconds())
ack := p.ack ack := pm.ack
if pc.messagePool {
messagePool.Put(p) messagePool.Put(p)
}
if ack { if ack {
pc.c.MarkCommitRecords(record) pc.c.MarkCommitRecords(record)
} else { } else {
if sp != nil {
sp.Finish() sp.Finish()
}
// pc.connected.Store(0) // pc.connected.Store(0)
pc.kopts.Logger.Fatal(pc.kopts.Context, "[kgo] message not commited") pc.kopts.Logger.Fatal(pc.kopts.Context, "[kgo] message not commited")
return return
} }
if sp != nil {
sp.Finish() sp.Finish()
} }
} }
} }
}
} }

View File

@@ -32,6 +32,9 @@ var (
// the record's context, so it can be ended in the OnProduceRecordUnbuffered // the record's context, so it can be ended in the OnProduceRecordUnbuffered
// hook. // hook.
func (m *hookTracer) OnProduceRecordBuffered(r *kgo.Record) { func (m *hookTracer) OnProduceRecordBuffered(r *kgo.Record) {
if !m.tracer.Enabled() {
return
}
// Set up span options. // Set up span options.
attrs := []interface{}{ attrs := []interface{}{
messagingSystem, messagingSystem,
@@ -77,6 +80,9 @@ func (m *hookTracer) OnProduceRecordBuffered(r *kgo.Record) {
// It sets attributes with values unset when producing and records any error // It sets attributes with values unset when producing and records any error
// that occurred during the publish operation. // that occurred during the publish operation.
func (m *hookTracer) OnProduceRecordUnbuffered(r *kgo.Record, err error) { func (m *hookTracer) OnProduceRecordUnbuffered(r *kgo.Record, err error) {
if !m.tracer.Enabled() {
return
}
if span, ok := tracer.SpanFromContext(r.Context); ok { if span, ok := tracer.SpanFromContext(r.Context); ok {
span.AddLabels( span.AddLabels(
semconv.MessagingKafkaDestinationPartition(int(r.Partition)), semconv.MessagingKafkaDestinationPartition(int(r.Partition)),
@@ -96,6 +102,9 @@ func (m *hookTracer) OnProduceRecordUnbuffered(r *kgo.Record, err error) {
// OnFetchRecordUnbuffered hook and can be used in downstream consumer // OnFetchRecordUnbuffered hook and can be used in downstream consumer
// processing. // processing.
func (m *hookTracer) OnFetchRecordBuffered(r *kgo.Record) { func (m *hookTracer) OnFetchRecordBuffered(r *kgo.Record) {
if !m.tracer.Enabled() {
return
}
// Set up the span options. // Set up the span options.
attrs := []interface{}{ attrs := []interface{}{
messagingSystem, messagingSystem,
@@ -141,6 +150,9 @@ func (m *hookTracer) OnFetchRecordBuffered(r *kgo.Record) {
// OnFetchRecordUnbuffered continues and ends the "receive" span for an // OnFetchRecordUnbuffered continues and ends the "receive" span for an
// unbuffered record. // unbuffered record.
func (m *hookTracer) OnFetchRecordUnbuffered(r *kgo.Record, _ bool) { func (m *hookTracer) OnFetchRecordUnbuffered(r *kgo.Record, _ bool) {
if !m.tracer.Enabled() {
return
}
span, _ := tracer.SpanFromContext(r.Context) span, _ := tracer.SpanFromContext(r.Context)
span.Finish() span.Finish()
} }
@@ -155,6 +167,13 @@ func (m *hookTracer) OnFetchRecordUnbuffered(r *kgo.Record, _ bool) {
// not a record which has been created for producing, so call this at the start of each // not a record which has been created for producing, so call this at the start of each
// iteration of your processing for the record. // iteration of your processing for the record.
func (m *hookTracer) WithProcessSpan(r *kgo.Record) (context.Context, tracer.Span) { func (m *hookTracer) WithProcessSpan(r *kgo.Record) (context.Context, tracer.Span) {
if r.Context == nil {
r.Context = context.Background()
}
if !m.tracer.Enabled() {
return r.Context, nil
}
// Set up the span options. // Set up the span options.
attrs := []interface{}{ attrs := []interface{}{
messagingSystem, messagingSystem,