Compare commits

...

9 Commits

Author SHA1 Message Date
4b0b70e7d0 fixup message options and content-type header
Some checks failed
coverage / build (push) Successful in 4m34s
test / test (push) Failing after 17m28s
Signed-off-by: Vasiliy Tolstov <v.tolstov@unistack.org>
2025-07-02 16:20:02 +03:00
vtolstov
da88718e03 Apply Code Coverage Badge 2025-06-30 13:11:01 +00:00
aed304b7e3 rework message options
Some checks failed
sync / sync (push) Successful in 1m2s
coverage / build (push) Successful in 3m31s
test / test (push) Failing after 18m28s
Signed-off-by: Vasiliy Tolstov <v.tolstov@unistack.org>
2025-06-30 16:08:59 +03:00
b796c43257 [v4] hide access to internal mutex (#185)
Some checks failed
coverage / build (push) Failing after 2m4s
test / test (push) Successful in 3m53s
sync / sync (push) Successful in 8s
* changed embedded mutex to private field

* update ci

* fix tests
2025-06-10 16:28:17 +03:00
b0691edf82 disable message pool by default
Signed-off-by: Vasiliy Tolstov <v.tolstov@unistack.org>
2025-06-10 16:28:16 +03:00
d43b59f2f3 disable message pool by default
Signed-off-by: Vasiliy Tolstov <v.tolstov@unistack.org>
2025-06-10 16:28:03 +03:00
f976516d6b new kgo version
update deps

fixup race conditions

add kfake usage

Signed-off-by: Vasiliy Tolstov <v.tolstov@unistack.org>
2025-06-10 16:27:40 +03:00
8f5c03ad60 [v4] hide access to internal mutex (#185)
* changed embedded mutex to private field

* update ci

* fix tests
2025-06-10 16:26:52 +03:00
vtolstov
9e9c1434ff Apply Code Coverage Badge 2025-06-09 14:30:14 +00:00
5 changed files with 67 additions and 68 deletions

View File

@@ -1,2 +1,2 @@
# micro-broker-kgo # micro-broker-kgo
![Coverage](https://img.shields.io/badge/Coverage-65.1%25-yellow) ![Coverage](https://img.shields.io/badge/Coverage-62.9%25-yellow)

12
go.sum
View File

@@ -20,16 +20,10 @@ github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRI
github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII= github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR38lUII=
github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o= github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o=
github.com/spf13/cast v1.8.0 h1:gEN9K4b8Xws4EX0+a0reLmhq8moKn7ntRlQYgjPeCDk=
github.com/spf13/cast v1.8.0/go.mod h1:ancEpBxwJDODSW/UG4rDrAqiKolqNNh2DX3mk86cAdo=
github.com/spf13/cast v1.9.2 h1:SsGfm7M8QOFtEzumm7UZrZdLLquNdzFYfIbEXntcFbE= github.com/spf13/cast v1.9.2 h1:SsGfm7M8QOFtEzumm7UZrZdLLquNdzFYfIbEXntcFbE=
github.com/spf13/cast v1.9.2/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo= github.com/spf13/cast v1.9.2/go.mod h1:jNfB8QC9IA6ZuY2ZjDp0KtFO2LZZlg4S/7bzP6qqeHo=
github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/twmb/franz-go v1.19.1 h1:cOhDFUkGvUFHSQ7UYW6bO77BJa2fYEk5mA2AX+1NIdE=
github.com/twmb/franz-go v1.19.1/go.mod h1:4kFJ5tmbbl7asgwAGVuyG1ZMx0NNpYk7EqflvWfPCpM=
github.com/twmb/franz-go v1.19.4 h1:0ktflzm5YU7+YYdie8RQWFcU9uDJ03xLefplO1iMwO4=
github.com/twmb/franz-go v1.19.4/go.mod h1:4kFJ5tmbbl7asgwAGVuyG1ZMx0NNpYk7EqflvWfPCpM=
github.com/twmb/franz-go v1.19.5 h1:W7+o8D0RsQsedqib71OVlLeZ0zI6CbFra7yTYhZTs5Y= github.com/twmb/franz-go v1.19.5 h1:W7+o8D0RsQsedqib71OVlLeZ0zI6CbFra7yTYhZTs5Y=
github.com/twmb/franz-go v1.19.5/go.mod h1:4kFJ5tmbbl7asgwAGVuyG1ZMx0NNpYk7EqflvWfPCpM= github.com/twmb/franz-go v1.19.5/go.mod h1:4kFJ5tmbbl7asgwAGVuyG1ZMx0NNpYk7EqflvWfPCpM=
github.com/twmb/franz-go/pkg/kadm v1.16.0 h1:STMs1t5lYR5mR974PSiwNzE5TvsosByTp+rKXLOhAjE= github.com/twmb/franz-go/pkg/kadm v1.16.0 h1:STMs1t5lYR5mR974PSiwNzE5TvsosByTp+rKXLOhAjE=
@@ -38,18 +32,12 @@ github.com/twmb/franz-go/pkg/kfake v0.0.0-20250508175730-72e1646135e3 h1:p24opKW
github.com/twmb/franz-go/pkg/kfake v0.0.0-20250508175730-72e1646135e3/go.mod h1:7uQs3Ae6HkWT1Y9elMbqtAcNFCI0y6+iS+Phw49L49U= github.com/twmb/franz-go/pkg/kfake v0.0.0-20250508175730-72e1646135e3/go.mod h1:7uQs3Ae6HkWT1Y9elMbqtAcNFCI0y6+iS+Phw49L49U=
github.com/twmb/franz-go/pkg/kmsg v1.11.2 h1:hIw75FpwcAjgeyfIGFqivAvwC5uNIOWRGvQgZhH4mhg= github.com/twmb/franz-go/pkg/kmsg v1.11.2 h1:hIw75FpwcAjgeyfIGFqivAvwC5uNIOWRGvQgZhH4mhg=
github.com/twmb/franz-go/pkg/kmsg v1.11.2/go.mod h1:CFfkkLysDNmukPYhGzuUcDtf46gQSqCZHMW1T4Z+wDE= github.com/twmb/franz-go/pkg/kmsg v1.11.2/go.mod h1:CFfkkLysDNmukPYhGzuUcDtf46gQSqCZHMW1T4Z+wDE=
go.opentelemetry.io/otel v1.35.0 h1:xKWKPxrxB6OtMCbmMY021CqC45J+3Onta9MqjhnusiQ=
go.opentelemetry.io/otel v1.35.0/go.mod h1:UEqy8Zp11hpkUrL73gSlELM0DupHoiq72dR+Zqel/+Y=
go.opentelemetry.io/otel v1.36.0 h1:UumtzIklRBY6cI/lllNZlALOF5nNIzJVb16APdvgTXg= go.opentelemetry.io/otel v1.36.0 h1:UumtzIklRBY6cI/lllNZlALOF5nNIzJVb16APdvgTXg=
go.opentelemetry.io/otel v1.36.0/go.mod h1:/TcFMXYjyRNh8khOAO9ybYkqaDBb/70aVwkNML4pP8E= go.opentelemetry.io/otel v1.36.0/go.mod h1:/TcFMXYjyRNh8khOAO9ybYkqaDBb/70aVwkNML4pP8E=
go.unistack.org/micro-proto/v4 v4.1.0 h1:qPwL2n/oqh9RE3RTTDgt28XK3QzV597VugQPaw9lKUk= go.unistack.org/micro-proto/v4 v4.1.0 h1:qPwL2n/oqh9RE3RTTDgt28XK3QzV597VugQPaw9lKUk=
go.unistack.org/micro-proto/v4 v4.1.0/go.mod h1:ArmK7o+uFvxSY3dbJhKBBX4Pm1rhWdLEFf3LxBrMtec= go.unistack.org/micro-proto/v4 v4.1.0/go.mod h1:ArmK7o+uFvxSY3dbJhKBBX4Pm1rhWdLEFf3LxBrMtec=
go.unistack.org/micro/v4 v4.1.14 h1:6EotPq9kz/gaFb5YulHdKuuUwmj/7Hk44DpOlzh/A6k=
go.unistack.org/micro/v4 v4.1.14/go.mod h1:xleO2M5Yxh4s6I+RUcLrEpUjobefh+71ctrdIfn7TUs=
go.unistack.org/micro/v4 v4.1.17 h1:26QDtRSYVpozYuassyvLP4sEQRo3dxgD3sVILRXmIPo= go.unistack.org/micro/v4 v4.1.17 h1:26QDtRSYVpozYuassyvLP4sEQRo3dxgD3sVILRXmIPo=
go.unistack.org/micro/v4 v4.1.17/go.mod h1:xleO2M5Yxh4s6I+RUcLrEpUjobefh+71ctrdIfn7TUs= go.unistack.org/micro/v4 v4.1.17/go.mod h1:xleO2M5Yxh4s6I+RUcLrEpUjobefh+71ctrdIfn7TUs=
golang.org/x/crypto v0.38.0 h1:jt+WWG8IZlBnVbomuhg2Mdq0+BBQaHbtqHEFEigjUV8=
golang.org/x/crypto v0.38.0/go.mod h1:MvrbAqul58NNYPKnOra203SB9vpuZW0e+RRZV+Ggqjw=
golang.org/x/crypto v0.39.0 h1:SHs+kF4LP+f+p14esP5jAoDpHU8Gu/v9lFRK6IT5imM= golang.org/x/crypto v0.39.0 h1:SHs+kF4LP+f+p14esP5jAoDpHU8Gu/v9lFRK6IT5imM=
golang.org/x/crypto v0.39.0/go.mod h1:L+Xg3Wf6HoL4Bn4238Z6ft6KfEpN0tJGo53AAPC632U= golang.org/x/crypto v0.39.0/go.mod h1:L+Xg3Wf6HoL4Bn4238Z6ft6KfEpN0tJGo53AAPC632U=
google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY= google.golang.org/protobuf v1.36.6 h1:z1NpPI8ku2WgiWnf+t9wTPsn6eP1L7ksHUlkfLvd9xY=

62
kgo.go
View File

@@ -156,7 +156,7 @@ func (b *Broker) NewMessage(ctx context.Context, hdr metadata.Metadata, body int
options.ContentType = b.opts.ContentType options.ContentType = b.opts.ContentType
} }
m := &kgoMessage{ctx: ctx, hdr: hdr, opts: options} m := &kgoMessage{ctx: ctx, hdr: hdr.Copy(), opts: options}
c, err := b.newCodec(m.opts.ContentType) c, err := b.newCodec(m.opts.ContentType)
if err == nil { if err == nil {
m.body, err = c.Marshal(body) m.body, err = c.Marshal(body)
@@ -165,6 +165,8 @@ func (b *Broker) NewMessage(ctx context.Context, hdr metadata.Metadata, body int
return nil, err return nil, err
} }
m.hdr.Set(metadata.HeaderContentType, m.opts.ContentType)
return m, nil return m, nil
} }
@@ -344,40 +346,44 @@ func (b *Broker) fnPublish(ctx context.Context, topic string, messages ...broker
} }
func (b *Broker) publish(ctx context.Context, topic string, messages ...broker.Message) error { func (b *Broker) publish(ctx context.Context, topic string, messages ...broker.Message) error {
records := make([]*kgo.Record, 0, len(messages)) var records []*kgo.Record
var errs []string
var key []byte
var promise func(*kgo.Record, error)
for _, msg := range messages { for _, msg := range messages {
if mctx := msg.Context(); mctx != nil {
if k, ok := mctx.Value(messageKey{}).([]byte); ok && k != nil {
key = k
}
if p, ok := mctx.Value(messagePromiseKey{}).(func(*kgo.Record, error)); ok && p != nil {
promise = p
}
}
rec := &kgo.Record{ rec := &kgo.Record{
Context: ctx, Context: msg.Context(),
Key: key,
Topic: topic, Topic: topic,
Value: msg.Body(), Value: msg.Body(),
} }
b.opts.Meter.Counter(semconv.PublishMessageInflight, "endpoint", rec.Topic, "topic", rec.Topic).Inc() var promise func(*kgo.Record, error)
if rec.Context != nil {
if k, ok := rec.Context.Value(messageKey{}).([]byte); ok && k != nil {
rec.Key = k
}
if p, ok := rec.Context.Value(messagePromiseKey{}).(func(*kgo.Record, error)); ok && p != nil {
promise = p
}
}
kmsg, ok := msg.(*kgoMessage)
if !ok {
continue
}
if kmsg.opts.Context != nil {
if k, ok := kmsg.opts.Context.Value(messageKey{}).([]byte); ok && k != nil {
rec.Key = k
}
if p, ok := kmsg.opts.Context.Value(messagePromiseKey{}).(func(*kgo.Record, error)); ok && p != nil {
promise = p
}
}
setHeaders(rec, msg.Header()) setHeaders(rec, msg.Header())
records = append(records, rec)
}
ts := time.Now()
if promise != nil { if promise != nil {
ts := time.Now()
for _, rec := range records { b.opts.Meter.Counter(semconv.PublishMessageInflight, "endpoint", rec.Topic, "topic", rec.Topic).Inc()
b.c.Produce(ctx, rec, func(r *kgo.Record, err error) { b.c.Produce(ctx, rec, func(r *kgo.Record, err error) {
te := time.Since(ts) te := time.Since(ts)
b.opts.Meter.Counter(semconv.PublishMessageInflight, "endpoint", rec.Topic, "topic", rec.Topic).Dec() b.opts.Meter.Counter(semconv.PublishMessageInflight, "endpoint", rec.Topic, "topic", rec.Topic).Dec()
@@ -390,12 +396,17 @@ func (b *Broker) publish(ctx context.Context, topic string, messages ...broker.M
} }
promise(r, err) promise(r, err)
}) })
continue
} else {
records = append(records, rec)
} }
return nil
} }
if len(records) > 0 {
var errs []string
ts := time.Now()
b.opts.Meter.Counter(semconv.PublishMessageInflight, "endpoint", topic, "topic", topic).Set(uint64(len(records)))
results := b.c.ProduceSync(ctx, records...) results := b.c.ProduceSync(ctx, records...)
te := time.Since(ts) te := time.Since(ts)
for _, result := range results { for _, result := range results {
b.opts.Meter.Summary(semconv.PublishMessageLatencyMicroseconds, "endpoint", result.Record.Topic, "topic", result.Record.Topic).Update(te.Seconds()) b.opts.Meter.Summary(semconv.PublishMessageLatencyMicroseconds, "endpoint", result.Record.Topic, "topic", result.Record.Topic).Update(te.Seconds())
@@ -412,6 +423,7 @@ func (b *Broker) publish(ctx context.Context, topic string, messages ...broker.M
if len(errs) > 0 { if len(errs) > 0 {
return fmt.Errorf("publish error: %s", strings.Join(errs, "\n")) return fmt.Errorf("publish error: %s", strings.Join(errs, "\n"))
} }
}
return nil return nil
} }

View File

@@ -81,7 +81,7 @@ func TestFail(t *testing.T) {
go func() { go func() {
for _, msg := range msgs { for _, msg := range msgs {
// t.Logf("broker publish") // t.Logf("broker publish")
if err := b.Publish(ctx, "test", msg); err != nil { if err := b.Publish(ctx, "test.fail", msg); err != nil {
t.Fatal(err) t.Fatal(err)
} }
} }
@@ -96,7 +96,7 @@ func TestFail(t *testing.T) {
return msg.Ack() return msg.Ack()
} }
sub, err := b.Subscribe(ctx, "test", fn, sub, err := b.Subscribe(ctx, "test.fail", fn,
broker.SubscribeAutoAck(true), broker.SubscribeAutoAck(true),
broker.SubscribeGroup(group), broker.SubscribeGroup(group),
broker.SubscribeBodyOnly(true)) broker.SubscribeBodyOnly(true))
@@ -184,7 +184,7 @@ func TestPubSub(t *testing.T) {
msgs = append(msgs, m) msgs = append(msgs, m)
} }
if err := b.Publish(ctx, "test", msgs...); err != nil { if err := b.Publish(ctx, "test.pubsub", msgs...); err != nil {
t.Fatal(err) t.Fatal(err)
} }
// t.Skip() // t.Skip()
@@ -197,7 +197,7 @@ func TestPubSub(t *testing.T) {
return msg.Ack() return msg.Ack()
} }
sub, err := b.Subscribe(ctx, "test", fn, sub, err := b.Subscribe(ctx, "test.pubsub", fn,
broker.SubscribeAutoAck(true), broker.SubscribeAutoAck(true),
broker.SubscribeGroup(group), broker.SubscribeGroup(group),
broker.SubscribeBodyOnly(true)) broker.SubscribeBodyOnly(true))

View File

@@ -48,7 +48,6 @@ type Subscriber struct {
done chan struct{} done chan struct{}
kopts broker.Options kopts broker.Options
opts broker.SubscribeOptions opts broker.SubscribeOptions
connected *atomic.Uint32 connected *atomic.Uint32
mu sync.RWMutex mu sync.RWMutex
closed bool closed bool