Compare commits
5 Commits
Author | SHA1 | Date | |
---|---|---|---|
969e459e3d | |||
|
b2cd7d8b8e | ||
6d85d3ee41 | |||
d0978fb314 | |||
|
0bd81d1253 |
11
README.md
11
README.md
@@ -1,9 +1,2 @@
|
|||||||
# broker-kgo
|
# micro-broker-kgo
|
||||||
|

|
||||||
TODO:
|
|
||||||
|
|
||||||
1) экспортируем текущий оффсет для каждой партиции в группе
|
|
||||||
2) экспортируем лаг для группы
|
|
||||||
3) мониторим
|
|
||||||
1) если есть лаг больше нуля
|
|
||||||
2) если дельта оффсета за нужное нам время не
|
|
||||||
|
3
go.mod
3
go.mod
@@ -8,8 +8,7 @@ require (
|
|||||||
github.com/twmb/franz-go/pkg/kfake v0.0.0-20250508175730-72e1646135e3
|
github.com/twmb/franz-go/pkg/kfake v0.0.0-20250508175730-72e1646135e3
|
||||||
github.com/twmb/franz-go/pkg/kmsg v1.11.2
|
github.com/twmb/franz-go/pkg/kmsg v1.11.2
|
||||||
go.opentelemetry.io/otel v1.35.0
|
go.opentelemetry.io/otel v1.35.0
|
||||||
go.unistack.org/micro-codec-json/v4 v4.1.0
|
go.unistack.org/micro/v4 v4.1.14
|
||||||
go.unistack.org/micro/v4 v4.1.13
|
|
||||||
)
|
)
|
||||||
|
|
||||||
require (
|
require (
|
||||||
|
8
go.sum
8
go.sum
@@ -24,8 +24,6 @@ github.com/spf13/cast v1.8.0 h1:gEN9K4b8Xws4EX0+a0reLmhq8moKn7ntRlQYgjPeCDk=
|
|||||||
github.com/spf13/cast v1.8.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo=
|
github.com/spf13/cast v1.8.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo=
|
||||||
github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
|
github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
|
||||||
github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
|
github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
|
||||||
github.com/twmb/franz-go v1.19.0 h1:FzBAPUeaip68X9cbLDesgQesa5zxKVaZMk+du98vj3c=
|
|
||||||
github.com/twmb/franz-go v1.19.0/go.mod h1:4kFJ5tmbbl7asgwAGVuyG1ZMx0NNpYk7EqflvWfPCpM=
|
|
||||||
github.com/twmb/franz-go v1.19.1 h1:cOhDFUkGvUFHSQ7UYW6bO77BJa2fYEk5mA2AX+1NIdE=
|
github.com/twmb/franz-go v1.19.1 h1:cOhDFUkGvUFHSQ7UYW6bO77BJa2fYEk5mA2AX+1NIdE=
|
||||||
github.com/twmb/franz-go v1.19.1/go.mod h1:4kFJ5tmbbl7asgwAGVuyG1ZMx0NNpYk7EqflvWfPCpM=
|
github.com/twmb/franz-go v1.19.1/go.mod h1:4kFJ5tmbbl7asgwAGVuyG1ZMx0NNpYk7EqflvWfPCpM=
|
||||||
github.com/twmb/franz-go/pkg/kadm v1.16.0 h1:STMs1t5lYR5mR974PSiwNzE5TvsosByTp+rKXLOhAjE=
|
github.com/twmb/franz-go/pkg/kadm v1.16.0 h1:STMs1t5lYR5mR974PSiwNzE5TvsosByTp+rKXLOhAjE=
|
||||||
@@ -36,12 +34,10 @@ github.com/twmb/franz-go/pkg/kmsg v1.11.2 h1:hIw75FpwcAjgeyfIGFqivAvwC5uNIOWRGvQ
|
|||||||
github.com/twmb/franz-go/pkg/kmsg v1.11.2/go.mod h1:CFfkkLysDNmukPYhGzuUcDtf46gQSqCZHMW1T4Z+wDE=
|
github.com/twmb/franz-go/pkg/kmsg v1.11.2/go.mod h1:CFfkkLysDNmukPYhGzuUcDtf46gQSqCZHMW1T4Z+wDE=
|
||||||
go.opentelemetry.io/otel v1.35.0 h1:xKWKPxrxB6OtMCbmMY021CqC45J+3Onta9MqjhnusiQ=
|
go.opentelemetry.io/otel v1.35.0 h1:xKWKPxrxB6OtMCbmMY021CqC45J+3Onta9MqjhnusiQ=
|
||||||
go.opentelemetry.io/otel v1.35.0/go.mod h1:UEqy8Zp11hpkUrL73gSlELM0DupHoiq72dR+Zqel/+Y=
|
go.opentelemetry.io/otel v1.35.0/go.mod h1:UEqy8Zp11hpkUrL73gSlELM0DupHoiq72dR+Zqel/+Y=
|
||||||
go.unistack.org/micro-codec-json/v4 v4.1.0 h1:iydeSkt3ee7IPU0dHHKlGN97lw+YFQasBk9rdv0woYA=
|
|
||||||
go.unistack.org/micro-codec-json/v4 v4.1.0/go.mod h1:aUg86elSlURSynTAetDAAXj/VzFDwwcg92QNrRzcvrM=
|
|
||||||
go.unistack.org/micro-proto/v4 v4.1.0 h1:qPwL2n/oqh9RE3RTTDgt28XK3QzV597VugQPaw9lKUk=
|
go.unistack.org/micro-proto/v4 v4.1.0 h1:qPwL2n/oqh9RE3RTTDgt28XK3QzV597VugQPaw9lKUk=
|
||||||
go.unistack.org/micro-proto/v4 v4.1.0/go.mod h1:ArmK7o+uFvxSY3dbJhKBBX4Pm1rhWdLEFf3LxBrMtec=
|
go.unistack.org/micro-proto/v4 v4.1.0/go.mod h1:ArmK7o+uFvxSY3dbJhKBBX4Pm1rhWdLEFf3LxBrMtec=
|
||||||
go.unistack.org/micro/v4 v4.1.13 h1:1IEQwiIwHdypZN4dWmsWN83Plq9bdkQ4U8aYZsT17s8=
|
go.unistack.org/micro/v4 v4.1.14 h1:6EotPq9kz/gaFb5YulHdKuuUwmj/7Hk44DpOlzh/A6k=
|
||||||
go.unistack.org/micro/v4 v4.1.13/go.mod h1:xleO2M5Yxh4s6I+RUcLrEpUjobefh+71ctrdIfn7TUs=
|
go.unistack.org/micro/v4 v4.1.14/go.mod h1:xleO2M5Yxh4s6I+RUcLrEpUjobefh+71ctrdIfn7TUs=
|
||||||
golang.org/x/crypto v0.38.0 h1:jt+WWG8IZlBnVbomuhg2Mdq0+BBQaHbtqHEFEigjUV8=
|
golang.org/x/crypto v0.38.0 h1:jt+WWG8IZlBnVbomuhg2Mdq0+BBQaHbtqHEFEigjUV8=
|
||||||
golang.org/x/crypto v0.38.0/go.mod h1:MvrbAqul58NNYPKnOra203SB9vpuZW0e+RRZV+Ggqjw=
|
golang.org/x/crypto v0.38.0/go.mod h1:MvrbAqul58NNYPKnOra203SB9vpuZW0e+RRZV+Ggqjw=
|
||||||
google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY=
|
google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY=
|
||||||
|
5
kgo.go
5
kgo.go
@@ -478,11 +478,15 @@ func (b *Broker) fnSubscribe(ctx context.Context, topic string, handler interfac
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
var messagePool bool
|
||||||
var fatalOnError bool
|
var fatalOnError bool
|
||||||
if b.opts.Context != nil {
|
if b.opts.Context != nil {
|
||||||
if v, ok := b.opts.Context.Value(fatalOnErrorKey{}).(bool); ok && v {
|
if v, ok := b.opts.Context.Value(fatalOnErrorKey{}).(bool); ok && v {
|
||||||
fatalOnError = v
|
fatalOnError = v
|
||||||
}
|
}
|
||||||
|
if v, ok := b.opts.Context.Value(subscribeMessagePoolKey{}).(bool); ok && v {
|
||||||
|
messagePool = v
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if options.Context != nil {
|
if options.Context != nil {
|
||||||
@@ -500,6 +504,7 @@ func (b *Broker) fnSubscribe(ctx context.Context, topic string, handler interfac
|
|||||||
done: make(chan struct{}),
|
done: make(chan struct{}),
|
||||||
fatalOnError: fatalOnError,
|
fatalOnError: fatalOnError,
|
||||||
connected: b.connected,
|
connected: b.connected,
|
||||||
|
messagePool: messagePool,
|
||||||
}
|
}
|
||||||
|
|
||||||
kopts := append(b.kopts,
|
kopts := append(b.kopts,
|
||||||
|
@@ -109,3 +109,10 @@ type publishPromiseKey struct{}
|
|||||||
func PublishPromise(fn func(*kgo.Record, error)) broker.PublishOption {
|
func PublishPromise(fn func(*kgo.Record, error)) broker.PublishOption {
|
||||||
return broker.SetPublishOption(publishPromiseKey{}, fn)
|
return broker.SetPublishOption(publishPromiseKey{}, fn)
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type subscribeMessagePoolKey struct{}
|
||||||
|
|
||||||
|
// SubscribeMessagePool optionaly enabled/disable message pool
|
||||||
|
func SubscribeMessagePool(b bool) broker.SubscribeOption {
|
||||||
|
return broker.SetSubscribeOption(subscribeMessagePoolKey{}, b)
|
||||||
|
}
|
||||||
|
@@ -35,6 +35,7 @@ type consumer struct {
|
|||||||
opts broker.SubscribeOptions
|
opts broker.SubscribeOptions
|
||||||
handler interface{}
|
handler interface{}
|
||||||
connected *atomic.Uint32
|
connected *atomic.Uint32
|
||||||
|
messagePool bool
|
||||||
}
|
}
|
||||||
|
|
||||||
type Subscriber struct {
|
type Subscriber struct {
|
||||||
@@ -42,7 +43,7 @@ type Subscriber struct {
|
|||||||
c *kgo.Client
|
c *kgo.Client
|
||||||
htracer *hookTracer
|
htracer *hookTracer
|
||||||
topic string
|
topic string
|
||||||
|
messagePool bool
|
||||||
handler interface{}
|
handler interface{}
|
||||||
done chan struct{}
|
done chan struct{}
|
||||||
kopts broker.Options
|
kopts broker.Options
|
||||||
@@ -224,6 +225,7 @@ func (s *Subscriber) assigned(_ context.Context, c *kgo.Client, assigned map[str
|
|||||||
done: make(chan struct{}),
|
done: make(chan struct{}),
|
||||||
recs: make(chan kgo.FetchTopicPartition, 100),
|
recs: make(chan kgo.FetchTopicPartition, 100),
|
||||||
handler: s.handler,
|
handler: s.handler,
|
||||||
|
messagePool: s.messagePool,
|
||||||
kopts: s.kopts,
|
kopts: s.kopts,
|
||||||
opts: s.opts,
|
opts: s.opts,
|
||||||
connected: s.connected,
|
connected: s.connected,
|
||||||
@@ -245,6 +247,8 @@ func (pc *consumer) consume() {
|
|||||||
defer pc.kopts.Logger.Debug(pc.kopts.Context, fmt.Sprintf("killing, topic %s partition %d", pc.topic, pc.partition))
|
defer pc.kopts.Logger.Debug(pc.kopts.Context, fmt.Sprintf("killing, topic %s partition %d", pc.topic, pc.partition))
|
||||||
}
|
}
|
||||||
|
|
||||||
|
var pm *kgoMessage
|
||||||
|
|
||||||
for {
|
for {
|
||||||
select {
|
select {
|
||||||
case <-pc.quit:
|
case <-pc.quit:
|
||||||
@@ -254,50 +258,60 @@ func (pc *consumer) consume() {
|
|||||||
ctx, sp := pc.htracer.WithProcessSpan(record)
|
ctx, sp := pc.htracer.WithProcessSpan(record)
|
||||||
ts := time.Now()
|
ts := time.Now()
|
||||||
pc.kopts.Meter.Counter(semconv.SubscribeMessageInflight, "endpoint", record.Topic, "topic", record.Topic).Inc()
|
pc.kopts.Meter.Counter(semconv.SubscribeMessageInflight, "endpoint", record.Topic, "topic", record.Topic).Inc()
|
||||||
p := messagePool.Get().(*kgoMessage)
|
|
||||||
|
|
||||||
p.body = record.Value
|
if pc.messagePool {
|
||||||
p.topic = record.Topic
|
pm = messagePool.Get().(*kgoMessage)
|
||||||
p.ack = false
|
} else {
|
||||||
p.hdr = metadata.New(len(record.Headers))
|
pm = &kgoMessage{}
|
||||||
p.ctx = ctx
|
}
|
||||||
|
pm.body = record.Value
|
||||||
|
pm.topic = record.Topic
|
||||||
|
pm.ack = false
|
||||||
|
pm.hdr = metadata.New(len(record.Headers))
|
||||||
|
pm.ctx = ctx
|
||||||
for _, hdr := range record.Headers {
|
for _, hdr := range record.Headers {
|
||||||
p.hdr.Set(hdr.Key, string(hdr.Value))
|
pm.hdr.Set(hdr.Key, string(hdr.Value))
|
||||||
}
|
}
|
||||||
|
|
||||||
switch h := pc.handler.(type) {
|
switch h := pc.handler.(type) {
|
||||||
case func(broker.Message) error:
|
case func(broker.Message) error:
|
||||||
err = h(p)
|
err = h(pm)
|
||||||
case func([]broker.Message) error:
|
case func([]broker.Message) error:
|
||||||
err = h([]broker.Message{p})
|
err = h([]broker.Message{pm})
|
||||||
}
|
}
|
||||||
|
|
||||||
pc.kopts.Meter.Counter(semconv.SubscribeMessageInflight, "endpoint", record.Topic, "topic", record.Topic).Dec()
|
pc.kopts.Meter.Counter(semconv.SubscribeMessageInflight, "endpoint", record.Topic, "topic", record.Topic).Dec()
|
||||||
if err != nil {
|
if err != nil {
|
||||||
|
if sp != nil {
|
||||||
sp.SetStatus(tracer.SpanStatusError, err.Error())
|
sp.SetStatus(tracer.SpanStatusError, err.Error())
|
||||||
|
}
|
||||||
pc.kopts.Meter.Counter(semconv.SubscribeMessageTotal, "endpoint", record.Topic, "topic", record.Topic, "status", "failure").Inc()
|
pc.kopts.Meter.Counter(semconv.SubscribeMessageTotal, "endpoint", record.Topic, "topic", record.Topic, "status", "failure").Inc()
|
||||||
} else if pc.opts.AutoAck {
|
} else if pc.opts.AutoAck {
|
||||||
p.ack = true
|
pm.ack = true
|
||||||
}
|
}
|
||||||
|
|
||||||
te := time.Since(ts)
|
te := time.Since(ts)
|
||||||
pc.kopts.Meter.Summary(semconv.SubscribeMessageLatencyMicroseconds, "endpoint", record.Topic, "topic", record.Topic).Update(te.Seconds())
|
pc.kopts.Meter.Summary(semconv.SubscribeMessageLatencyMicroseconds, "endpoint", record.Topic, "topic", record.Topic).Update(te.Seconds())
|
||||||
pc.kopts.Meter.Histogram(semconv.SubscribeMessageDurationSeconds, "endpoint", record.Topic, "topic", record.Topic).Update(te.Seconds())
|
pc.kopts.Meter.Histogram(semconv.SubscribeMessageDurationSeconds, "endpoint", record.Topic, "topic", record.Topic).Update(te.Seconds())
|
||||||
|
|
||||||
ack := p.ack
|
ack := pm.ack
|
||||||
|
if pc.messagePool {
|
||||||
messagePool.Put(p)
|
messagePool.Put(p)
|
||||||
|
}
|
||||||
if ack {
|
if ack {
|
||||||
pc.c.MarkCommitRecords(record)
|
pc.c.MarkCommitRecords(record)
|
||||||
} else {
|
} else {
|
||||||
|
if sp != nil {
|
||||||
sp.Finish()
|
sp.Finish()
|
||||||
|
}
|
||||||
// pc.connected.Store(0)
|
// pc.connected.Store(0)
|
||||||
pc.kopts.Logger.Fatal(pc.kopts.Context, "[kgo] message not commited")
|
pc.kopts.Logger.Fatal(pc.kopts.Context, "[kgo] message not commited")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
|
if sp != nil {
|
||||||
sp.Finish()
|
sp.Finish()
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
}
|
||||||
|
19
tracer.go
19
tracer.go
@@ -32,6 +32,9 @@ var (
|
|||||||
// the record's context, so it can be ended in the OnProduceRecordUnbuffered
|
// the record's context, so it can be ended in the OnProduceRecordUnbuffered
|
||||||
// hook.
|
// hook.
|
||||||
func (m *hookTracer) OnProduceRecordBuffered(r *kgo.Record) {
|
func (m *hookTracer) OnProduceRecordBuffered(r *kgo.Record) {
|
||||||
|
if !m.tracer.Enabled() {
|
||||||
|
return
|
||||||
|
}
|
||||||
// Set up span options.
|
// Set up span options.
|
||||||
attrs := []interface{}{
|
attrs := []interface{}{
|
||||||
messagingSystem,
|
messagingSystem,
|
||||||
@@ -77,6 +80,9 @@ func (m *hookTracer) OnProduceRecordBuffered(r *kgo.Record) {
|
|||||||
// It sets attributes with values unset when producing and records any error
|
// It sets attributes with values unset when producing and records any error
|
||||||
// that occurred during the publish operation.
|
// that occurred during the publish operation.
|
||||||
func (m *hookTracer) OnProduceRecordUnbuffered(r *kgo.Record, err error) {
|
func (m *hookTracer) OnProduceRecordUnbuffered(r *kgo.Record, err error) {
|
||||||
|
if !m.tracer.Enabled() {
|
||||||
|
return
|
||||||
|
}
|
||||||
if span, ok := tracer.SpanFromContext(r.Context); ok {
|
if span, ok := tracer.SpanFromContext(r.Context); ok {
|
||||||
span.AddLabels(
|
span.AddLabels(
|
||||||
semconv.MessagingKafkaDestinationPartition(int(r.Partition)),
|
semconv.MessagingKafkaDestinationPartition(int(r.Partition)),
|
||||||
@@ -96,6 +102,9 @@ func (m *hookTracer) OnProduceRecordUnbuffered(r *kgo.Record, err error) {
|
|||||||
// OnFetchRecordUnbuffered hook and can be used in downstream consumer
|
// OnFetchRecordUnbuffered hook and can be used in downstream consumer
|
||||||
// processing.
|
// processing.
|
||||||
func (m *hookTracer) OnFetchRecordBuffered(r *kgo.Record) {
|
func (m *hookTracer) OnFetchRecordBuffered(r *kgo.Record) {
|
||||||
|
if !m.tracer.Enabled() {
|
||||||
|
return
|
||||||
|
}
|
||||||
// Set up the span options.
|
// Set up the span options.
|
||||||
attrs := []interface{}{
|
attrs := []interface{}{
|
||||||
messagingSystem,
|
messagingSystem,
|
||||||
@@ -141,6 +150,9 @@ func (m *hookTracer) OnFetchRecordBuffered(r *kgo.Record) {
|
|||||||
// OnFetchRecordUnbuffered continues and ends the "receive" span for an
|
// OnFetchRecordUnbuffered continues and ends the "receive" span for an
|
||||||
// unbuffered record.
|
// unbuffered record.
|
||||||
func (m *hookTracer) OnFetchRecordUnbuffered(r *kgo.Record, _ bool) {
|
func (m *hookTracer) OnFetchRecordUnbuffered(r *kgo.Record, _ bool) {
|
||||||
|
if !m.tracer.Enabled() {
|
||||||
|
return
|
||||||
|
}
|
||||||
span, _ := tracer.SpanFromContext(r.Context)
|
span, _ := tracer.SpanFromContext(r.Context)
|
||||||
span.Finish()
|
span.Finish()
|
||||||
}
|
}
|
||||||
@@ -155,6 +167,13 @@ func (m *hookTracer) OnFetchRecordUnbuffered(r *kgo.Record, _ bool) {
|
|||||||
// not a record which has been created for producing, so call this at the start of each
|
// not a record which has been created for producing, so call this at the start of each
|
||||||
// iteration of your processing for the record.
|
// iteration of your processing for the record.
|
||||||
func (m *hookTracer) WithProcessSpan(r *kgo.Record) (context.Context, tracer.Span) {
|
func (m *hookTracer) WithProcessSpan(r *kgo.Record) (context.Context, tracer.Span) {
|
||||||
|
if r.Context == nil {
|
||||||
|
r.Context = context.Background()
|
||||||
|
}
|
||||||
|
|
||||||
|
if !m.tracer.Enabled() {
|
||||||
|
return r.Context, nil
|
||||||
|
}
|
||||||
// Set up the span options.
|
// Set up the span options.
|
||||||
attrs := []interface{}{
|
attrs := []interface{}{
|
||||||
messagingSystem,
|
messagingSystem,
|
||||||
|
Reference in New Issue
Block a user