update deps && structs && hooks
Some checks failed
codeql / analyze (go) (pull_request) Has been cancelled
prbuild / test (pull_request) Has been cancelled
prbuild / lint (pull_request) Has been cancelled

This commit is contained in:
Денис Евстигнеев 2024-12-19 13:21:32 +03:00
parent 9c4d88bb69
commit 1018abe7bd
11 changed files with 177 additions and 112 deletions

1
.gitignore vendored
View File

@ -13,3 +13,4 @@
# Dependency directories (remove the comment below to include it) # Dependency directories (remove the comment below to include it)
# vendor/ # vendor/
.idea

22
.golangci.yml Normal file
View File

@ -0,0 +1,22 @@
run:
concurrency: 8
deadline: 5m
issues-exit-code: 1
tests: true
linters:
enable:
- staticcheck
- unused
- gosimple
- govet
- goimports
- prealloc
- unconvert
- nakedret
linters-settings:
govet:
check-all: true
enable:
- fieldalignment

View File

@ -1,19 +1,27 @@
package kgo package kgo
import ( import (
"context"
"sync" "sync"
"go.unistack.org/micro/v3/broker" "go.unistack.org/micro/v3/broker"
) )
type event struct { type event struct {
topic string
err error
sync.RWMutex
msg *broker.Message msg *broker.Message
err error
topic string
sync.RWMutex
ack bool ack bool
} }
func (p *event) Context() context.Context {
//TODO implement me
return context.Background()
}
func (p *event) Topic() string { func (p *event) Topic() string {
return p.topic return p.topic
} }

25
go.mod
View File

@ -1,21 +1,24 @@
module go.unistack.org/micro-broker-kgo/v3 module go.unistack.org/micro-broker-kgo/v3
go 1.19 go 1.22.0
toolchain go1.23.3
require ( require (
github.com/google/uuid v1.6.0 github.com/google/uuid v1.6.0
github.com/twmb/franz-go v1.16.1 github.com/twmb/franz-go v1.18.0
github.com/twmb/franz-go/pkg/kmsg v1.7.0 github.com/twmb/franz-go/pkg/kmsg v1.9.0
go.opentelemetry.io/otel v1.24.0 go.opentelemetry.io/otel v1.33.0
go.unistack.org/micro/v3 v3.10.46 go.unistack.org/micro/v3 v3.11.20
) )
require ( require (
github.com/golang/protobuf v1.5.4 // indirect github.com/klauspost/compress v1.17.8 // indirect
github.com/klauspost/compress v1.17.7 // indirect
github.com/pierrec/lz4/v4 v4.1.21 // indirect github.com/pierrec/lz4/v4 v4.1.21 // indirect
golang.org/x/sys v0.18.0 // indirect go.unistack.org/micro-proto/v3 v3.4.1 // indirect
google.golang.org/genproto/googleapis/rpc v0.0.0-20240304212257-790db918fca8 // indirect golang.org/x/crypto v0.31.0 // indirect
google.golang.org/grpc v1.62.1 // indirect golang.org/x/sys v0.28.0 // indirect
google.golang.org/protobuf v1.33.0 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20241216192217-9240e9c98484 // indirect
google.golang.org/grpc v1.69.2 // indirect
google.golang.org/protobuf v1.36.0 // indirect
) )

58
go.sum
View File

@ -1,32 +1,42 @@
github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc h1:U9qPSI2PIWSS1VwoXQT9A3Wy9MM3WgvqSxFWenqJduM=
github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek= github.com/golang/protobuf v1.5.4 h1:i7eJL8qZTpSEXOPTxNKhASYpMn+8e5Q6AdndVa1dWek=
github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps= github.com/golang/protobuf v1.5.4/go.mod h1:lnTiLA8Wa4RWRcIUkrtSVa5nRhsEGBg48fD6rSs7xps=
github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/klauspost/compress v1.17.7 h1:ehO88t2UGzQK66LMdE8tibEd1ErmzZjNEqWkjLAKQQg= github.com/klauspost/compress v1.17.8 h1:YcnTYrq7MikUT7k0Yb5eceMmALQPYBW/Xltxn0NAMnU=
github.com/klauspost/compress v1.17.7/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw= github.com/klauspost/compress v1.17.8/go.mod h1:Di0epgTjJY877eYKx5yC51cX2A2Vl2ibi7bDH9ttBbw=
github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ= github.com/pierrec/lz4/v4 v4.1.21 h1:yOVMLb6qSIDP67pl/5F7RepeKYu/VmTyEXvuMI5d9mQ=
github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pierrec/lz4/v4 v4.1.21/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 h1:Jamvg5psRIccs7FGNTlIRMkT8wgtp5eCXdBlqhYGL6U=
github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
github.com/twmb/franz-go v1.16.1 h1:rpWc7fB9jd7TgmCyfxzenBI+QbgS8ZfJOUQE+tzPtbE= github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
github.com/twmb/franz-go v1.16.1/go.mod h1:/pER254UPPGp/4WfGqRi+SIRGE50RSQzVubQp6+N4FA= github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
github.com/twmb/franz-go/pkg/kmsg v1.7.0 h1:a457IbvezYfA5UkiBvyV3zj0Is3y1i8EJgqjJYoij2E= github.com/twmb/franz-go v1.18.0 h1:25FjMZfdozBywVX+5xrWC2W+W76i0xykKjTdEeD2ejw=
github.com/twmb/franz-go/pkg/kmsg v1.7.0/go.mod h1:se9Mjdt0Nwzc9lnjJ0HyDtLyBnaBDAd7pCje47OhSyw= github.com/twmb/franz-go v1.18.0/go.mod h1:zXCGy74M0p5FbXsLeASdyvfLFsBvTubVqctIaa5wQ+I=
go.opentelemetry.io/otel v1.24.0 h1:0LAOdjNmQeSTzGBzduGe/rU4tZhMwL5rWgtp9Ku5Jfo= github.com/twmb/franz-go/pkg/kmsg v1.9.0 h1:JojYUph2TKAau6SBtErXpXGC7E3gg4vGZMv9xFU/B6M=
go.opentelemetry.io/otel v1.24.0/go.mod h1:W7b9Ozg4nkF5tWI5zsXkaKKDjdVjpD4oAt9Qi/MArHo= github.com/twmb/franz-go/pkg/kmsg v1.9.0/go.mod h1:CMbfazviCyY6HM0SXuG5t9vOwYDHRCSrJJyBAe5paqg=
go.unistack.org/micro/v3 v3.10.46 h1:rnuEqiFkerwJmKzHmHBXRgxFemZustxWz2hRNLQQ8cU= go.opentelemetry.io/otel v1.33.0 h1:/FerN9bax5LoK51X/sI0SVYrjSE0/yUL7DpxW4K3FWw=
go.unistack.org/micro/v3 v3.10.46/go.mod h1:erMgt3Bl7vQQ0e9UpQyR5NlLiZ9pKeEJ9+1tfYFaqUg= go.opentelemetry.io/otel v1.33.0/go.mod h1:SUUkR6csvUQl+yjReHu5uM3EtVV7MBm5FHKRlNx4I8I=
golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= go.unistack.org/micro-proto/v3 v3.4.1 h1:UTjLSRz2YZuaHk9iSlVqqsA50JQNAEK2ZFboGqtEa9Q=
golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo= go.unistack.org/micro-proto/v3 v3.4.1/go.mod h1:okx/cnOhzuCX0ggl/vToatbCupi0O44diiiLLsZ93Zo=
golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= go.unistack.org/micro/v3 v3.11.20 h1:Foymuty1Oi1AqbTD/aKSo/0mAW3Az4SPqgPA5FqZChw=
golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= go.unistack.org/micro/v3 v3.11.20/go.mod h1:TjF2+KJ2RG+IB4d0wnXtaF5KgqwAqy/AMh+w9gDpRHg=
golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U=
google.golang.org/genproto/googleapis/rpc v0.0.0-20240304212257-790db918fca8 h1:IR+hp6ypxjH24bkMfEJ0yHR21+gwPWdV+/IBrPQyn3k= golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
google.golang.org/genproto/googleapis/rpc v0.0.0-20240304212257-790db918fca8/go.mod h1:UCOku4NytXMJuLQE5VuqA5lX3PcHCBo8pxNyvkf4xBs= golang.org/x/net v0.32.0 h1:ZqPmj8Kzc+Y6e0+skZsuACbx+wzMgo5MQsJh9Qd6aYI=
google.golang.org/grpc v1.62.1 h1:B4n+nfKzOICUXMgyrNd19h/I9oH0L1pizfk1d4zSgTk= golang.org/x/net v0.32.0/go.mod h1:CwU0IoeOlnQQWJ6ioyFrfRuomB8GKF6KbYXZVyeXNfs=
google.golang.org/grpc v1.62.1/go.mod h1:IWTG0VlJLCh1SkC58F7np9ka9mx/WNkjl4PGJaiq+QE= golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA=
google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo=
golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
google.golang.org/genproto/googleapis/rpc v0.0.0-20241216192217-9240e9c98484 h1:Z7FRVJPSMaHQxD0uXU8WdgFh8PseLM8Q8NzhnpMrBhQ=
google.golang.org/genproto/googleapis/rpc v0.0.0-20241216192217-9240e9c98484/go.mod h1:lcTa1sDdWEIHMWlITnIczmw5w60CF9ffkb8Z+DVmmjA=
google.golang.org/grpc v1.69.2 h1:U3S9QEtbXC0bYNvRtcoklF3xGtLViumSYxWykJS+7AU=
google.golang.org/grpc v1.69.2/go.mod h1:vyjdE6jLBI76dgpDojsFGNaHlxdjXN9ghpnd2o7JGZ4=
google.golang.org/protobuf v1.36.0 h1:mjIs9gYtt56AzC4ZaffQuh88TZurBGhIJMBZGSxNerQ=
google.golang.org/protobuf v1.36.0/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

53
kgo.go
View File

@ -5,7 +5,7 @@ import (
"context" "context"
"errors" "errors"
"fmt" "fmt"
"math/rand" "math/rand/v2"
"strings" "strings"
"sync" "sync"
"time" "time"
@ -25,7 +25,6 @@ var ErrLostMessage = errors.New("message not marked for offsets commit and will
var DefaultRetryBackoffFn = func() func(int) time.Duration { var DefaultRetryBackoffFn = func() func(int) time.Duration {
var rngMu sync.Mutex var rngMu sync.Mutex
rng := rand.New(rand.NewSource(time.Now().UnixNano()))
return func(fails int) time.Duration { return func(fails int) time.Duration {
const ( const (
min = 100 * time.Millisecond min = 100 * time.Millisecond
@ -41,7 +40,7 @@ var DefaultRetryBackoffFn = func() func(int) time.Duration {
backoff := min * time.Duration(1<<(fails-1)) backoff := min * time.Duration(1<<(fails-1))
rngMu.Lock() rngMu.Lock()
jitter := 0.8 + 0.4*rng.Float64() jitter := 0.8 + 0.4*rand.Float64()
rngMu.Unlock() rngMu.Unlock()
backoff = time.Duration(float64(backoff) * jitter) backoff = time.Duration(float64(backoff) * jitter)
@ -54,13 +53,31 @@ var DefaultRetryBackoffFn = func() func(int) time.Duration {
}() }()
type Broker struct { type Broker struct {
init bool c *kgo.Client
c *kgo.Client
kopts []kgo.Opt kopts []kgo.Opt
connected bool subs []*subscriber
sync.RWMutex
opts broker.Options opts broker.Options
subs []*subscriber
sync.RWMutex
connected bool
init bool
}
func (k *Broker) Live() bool {
//TODO implement me
return true
}
func (k *Broker) Ready() bool {
//TODO implement me
return true
}
func (k *Broker) Health() bool {
//TODO implement me
return true
} }
func (k *Broker) Address() string { func (k *Broker) Address() string {
@ -288,11 +305,11 @@ func (k *Broker) publish(ctx context.Context, msgs []*broker.Message, opts ...br
// k.opts.Meter.Counter(broker.PublishMessageInflight, "endpoint", rec.Topic).Dec() // k.opts.Meter.Counter(broker.PublishMessageInflight, "endpoint", rec.Topic).Dec()
// k.opts.Meter.Summary(broker.PublishMessageLatencyMicroseconds, "endpoint", r.Topic).Update(te.Seconds()) // k.opts.Meter.Summary(broker.PublishMessageLatencyMicroseconds, "endpoint", r.Topic).Update(te.Seconds())
// k.opts.Meter.Histogram(broker.PublishMessageDurationSeconds, "endpoint", r.Topic).Update(te.Seconds()) // k.opts.Meter.Histogram(broker.PublishMessageDurationSeconds, "endpoint", r.Topic).Update(te.Seconds())
if err != nil { // if err != nil {
// k.opts.Meter.Counter(broker.PublishMessageTotal, "endpoint", r.Topic, "status", "failure").Inc() // k.opts.Meter.Counter(broker.PublishMessageTotal, "endpoint", r.Topic, "status", "failure").Inc()
} else { // } else {
// k.opts.Meter.Counter(broker.PublishMessageTotal, "endpoint", r.Topic, "status", "success").Inc() // k.opts.Meter.Counter(broker.PublishMessageTotal, "endpoint", r.Topic, "status", "success").Inc()
} // }
promise(r, err) promise(r, err)
}) })
} }
@ -308,9 +325,9 @@ func (k *Broker) publish(ctx context.Context, msgs []*broker.Message, opts ...br
if result.Err != nil { if result.Err != nil {
// k.opts.Meter.Counter(broker.PublishMessageTotal, "endpoint", result.Record.Topic, "status", "failure").Inc() // k.opts.Meter.Counter(broker.PublishMessageTotal, "endpoint", result.Record.Topic, "status", "failure").Inc()
errs = append(errs, result.Err.Error()) errs = append(errs, result.Err.Error())
} else { } // else {
// k.opts.Meter.Counter(broker.PublishMessageTotal, "endpoint", result.Record.Topic, "status", "success").Inc() // k.opts.Meter.Counter(broker.PublishMessageTotal, "endpoint", result.Record.Topic, "status", "success").Inc()
} // }
} }
if len(errs) > 0 { if len(errs) > 0 {
@ -320,7 +337,7 @@ func (k *Broker) publish(ctx context.Context, msgs []*broker.Message, opts ...br
return nil return nil
} }
func (k *Broker) BatchSubscribe(ctx context.Context, topic string, handler broker.BatchHandler, opts ...broker.SubscribeOption) (broker.Subscriber, error) { func (k *Broker) BatchSubscribe(_ context.Context, _ string, _ broker.BatchHandler, _ ...broker.SubscribeOption) (broker.Subscriber, error) {
return nil, nil return nil, nil
} }

View File

@ -50,7 +50,7 @@ func TestPubSub(t *testing.T) {
t.Skip() t.Skip()
} }
if err := logger.DefaultLogger.Init(logger.WithLevel(loglevel), logger.WithCallerSkipCount(3)); err != nil { if err := logger.DefaultLogger.Init(logger.WithLevel(loglevel), logger.WithAddCallerSkipCount(1)); err != nil {
t.Fatal(err) t.Fatal(err)
} }
ctx := context.Background() ctx := context.Background()

View File

@ -28,11 +28,8 @@ func (l *mlogger) Log(lvl kgo.LogLevel, msg string, args ...interface{}) {
default: default:
return return
} }
if len(args) > 0 {
l.l.Log(l.ctx, mlvl, append([]interface{}{msg}, args...)...) l.l.Log(l.ctx, mlvl, msg, args...)
} else {
l.l.Log(l.ctx, mlvl, msg)
}
} }
func (l *mlogger) Level() kgo.LogLevel { func (l *mlogger) Level() kgo.LogLevel {

View File

@ -54,7 +54,7 @@ const (
labelTopic = "topic" labelTopic = "topic"
) )
func (m *hookMeter) OnGroupManageError(err error) { func (m *hookMeter) OnGroupManageError(_ error) {
m.meter.Counter(metricBrokerGroupErrors).Inc() m.meter.Counter(metricBrokerGroupErrors).Inc()
} }

View File

@ -2,6 +2,7 @@ package kgo
import ( import (
"context" "context"
"fmt"
"sync" "sync"
"github.com/twmb/franz-go/pkg/kgo" "github.com/twmb/franz-go/pkg/kgo"
@ -16,27 +17,33 @@ type tp struct {
} }
type consumer struct { type consumer struct {
c *kgo.Client topic string
topic string
c *kgo.Client
handler broker.Handler
quit chan struct{}
done chan struct{}
recs chan kgo.FetchTopicPartition
kopts broker.Options
opts broker.SubscribeOptions
partition int32 partition int32
opts broker.SubscribeOptions
kopts broker.Options
handler broker.Handler
quit chan struct{}
done chan struct{}
recs chan kgo.FetchTopicPartition
} }
type subscriber struct { type subscriber struct {
consumers map[tp]*consumer
c *kgo.Client c *kgo.Client
topic string topic string
opts broker.SubscribeOptions
kopts broker.Options handler broker.Handler
handler broker.Handler done chan struct{}
closed bool kopts broker.Options
done chan struct{} opts broker.SubscribeOptions
consumers map[tp]*consumer
sync.RWMutex sync.RWMutex
closed bool
} }
func (s *subscriber) Options() broker.SubscribeOptions { func (s *subscriber) Options() broker.SubscribeOptions {
@ -51,21 +58,21 @@ func (s *subscriber) Unsubscribe(ctx context.Context) error {
if s.closed { if s.closed {
return nil return nil
} }
select { // select {
// case <-ctx.Done(): // case <-ctx.Done():
// return ctx.Err() // return ctx.Err()
default: // default:
s.c.PauseFetchTopics(s.topic) s.c.PauseFetchTopics(s.topic)
s.c.CloseAllowingRebalance() s.c.CloseAllowingRebalance()
kc := make(map[string][]int32) kc := make(map[string][]int32)
for ctp := range s.consumers { for ctp := range s.consumers {
kc[ctp.t] = append(kc[ctp.t], ctp.p) kc[ctp.t] = append(kc[ctp.t], ctp.p)
}
s.killConsumers(ctx, kc)
close(s.done)
s.closed = true
s.c.ResumeFetchTopics(s.topic)
} }
s.killConsumers(ctx, kc)
close(s.done)
s.closed = true
s.c.ResumeFetchTopics(s.topic)
// }
return nil return nil
} }
@ -90,12 +97,12 @@ func (s *subscriber) poll(ctx context.Context) {
return return
} }
fetches.EachError(func(t string, p int32, err error) { fetches.EachError(func(t string, p int32, err error) {
s.kopts.Logger.Fatalf(ctx, "[kgo] fetch topic %s partition %d err: %v", t, p, err) s.kopts.Logger.Fatal(ctx, fmt.Sprintf("[kgo] fetch topic %s partition %d", t, p), err)
}) })
fetches.EachPartition(func(p kgo.FetchTopicPartition) { fetches.EachPartition(func(p kgo.FetchTopicPartition) {
tp := tp{p.Topic, p.Partition} nTp := tp{p.Topic, p.Partition}
s.consumers[tp].recs <- p s.consumers[nTp].recs <- p
}) })
s.c.AllowRebalance() s.c.AllowRebalance()
} }
@ -108,11 +115,11 @@ func (s *subscriber) killConsumers(ctx context.Context, lost map[string][]int32)
for topic, partitions := range lost { for topic, partitions := range lost {
for _, partition := range partitions { for _, partition := range partitions {
tp := tp{topic, partition} nTp := tp{topic, partition}
pc := s.consumers[tp] pc := s.consumers[nTp]
delete(s.consumers, tp) delete(s.consumers, nTp)
close(pc.quit) close(pc.quit)
s.kopts.Logger.Debugf(ctx, "[kgo] waiting for work to finish topic %s partition %d", topic, partition) s.kopts.Logger.Debug(ctx, fmt.Sprintf("[kgo] waiting for work to finish topic %s partition %d", topic, partition))
wg.Add(1) wg.Add(1)
go func() { <-pc.done; wg.Done() }() go func() { <-pc.done; wg.Done() }()
} }
@ -120,15 +127,15 @@ func (s *subscriber) killConsumers(ctx context.Context, lost map[string][]int32)
} }
func (s *subscriber) lost(ctx context.Context, _ *kgo.Client, lost map[string][]int32) { func (s *subscriber) lost(ctx context.Context, _ *kgo.Client, lost map[string][]int32) {
s.kopts.Logger.Debugf(ctx, "[kgo] lost %#+v", lost) s.kopts.Logger.Debug(ctx, fmt.Sprintf("[kgo] lost %#+v", lost))
s.killConsumers(ctx, lost) s.killConsumers(ctx, lost)
} }
func (s *subscriber) revoked(ctx context.Context, c *kgo.Client, revoked map[string][]int32) { func (s *subscriber) revoked(ctx context.Context, c *kgo.Client, revoked map[string][]int32) {
s.kopts.Logger.Debugf(ctx, "[kgo] revoked %#+v", revoked) s.kopts.Logger.Debug(ctx, fmt.Sprintf("[kgo] revoked %#+v", revoked))
s.killConsumers(ctx, revoked) s.killConsumers(ctx, revoked)
if err := c.CommitMarkedOffsets(ctx); err != nil { if err := c.CommitMarkedOffsets(ctx); err != nil {
s.kopts.Logger.Errorf(ctx, "[kgo] revoked CommitMarkedOffsets err: %v", err) s.kopts.Logger.Error(ctx, "[kgo] revoked CommitMarkedOffsets", err)
} }
} }
@ -155,8 +162,8 @@ func (s *subscriber) assigned(_ context.Context, c *kgo.Client, assigned map[str
func (pc *consumer) consume() { func (pc *consumer) consume() {
defer close(pc.done) defer close(pc.done)
pc.kopts.Logger.Debugf(pc.kopts.Context, "starting, topic %s partition %d", pc.topic, pc.partition) pc.kopts.Logger.Debug(pc.kopts.Context, fmt.Sprintf("starting, topic %s partition %d", pc.topic, pc.partition))
defer pc.kopts.Logger.Debugf(pc.kopts.Context, "killing, topic %s partition %d", pc.topic, pc.partition) defer pc.kopts.Logger.Debug(pc.kopts.Context, fmt.Sprintf("killing, topic %s partition %d", pc.topic, pc.partition))
eh := pc.kopts.ErrorHandler eh := pc.kopts.ErrorHandler
if pc.opts.ErrorHandler != nil { if pc.opts.ErrorHandler != nil {
@ -197,7 +204,7 @@ func (pc *consumer) consume() {
pc.c.MarkCommitRecords(record) pc.c.MarkCommitRecords(record)
} else { } else {
eventPool.Put(p) eventPool.Put(p)
pc.kopts.Logger.Fatalf(pc.kopts.Context, "[kgo] ErrLostMessage wtf?") pc.kopts.Logger.Fatal(pc.kopts.Context, "[kgo] ErrLostMessage wtf?")
return return
} }
eventPool.Put(p) eventPool.Put(p)
@ -207,7 +214,7 @@ func (pc *consumer) consume() {
continue continue
} else { } else {
if pc.kopts.Logger.V(logger.ErrorLevel) { if pc.kopts.Logger.V(logger.ErrorLevel) {
pc.kopts.Logger.Errorf(pc.kopts.Context, "[kgo]: failed to unmarshal: %v", err) pc.kopts.Logger.Error(pc.kopts.Context, "[kgo]: failed to unmarshal", err)
} }
} }
// te := time.Since(ts) // te := time.Since(ts)
@ -215,16 +222,16 @@ func (pc *consumer) consume() {
// pc.kopts.Meter.Summary(broker.SubscribeMessageLatencyMicroseconds, "endpoint", record.Topic).Update(te.Seconds()) // pc.kopts.Meter.Summary(broker.SubscribeMessageLatencyMicroseconds, "endpoint", record.Topic).Update(te.Seconds())
// pc.kopts.Meter.Histogram(broker.SubscribeMessageDurationSeconds, "endpoint", record.Topic).Update(te.Seconds()) // pc.kopts.Meter.Histogram(broker.SubscribeMessageDurationSeconds, "endpoint", record.Topic).Update(te.Seconds())
eventPool.Put(p) eventPool.Put(p)
pc.kopts.Logger.Fatalf(pc.kopts.Context, "[kgo] Unmarshal err not handled wtf?") pc.kopts.Logger.Fatal(pc.kopts.Context, "[kgo] Unmarshal err not handled wtf?")
return return
} }
} }
err := pc.handler(p) err := pc.handler(p)
if err == nil { // if err == nil {
// pc.kopts.Meter.Counter(broker.SubscribeMessageTotal, "endpoint", record.Topic, "status", "success").Inc() // pc.kopts.Meter.Counter(broker.SubscribeMessageTotal, "endpoint", record.Topic, "status", "success").Inc()
} else { // } else {
// pc.kopts.Meter.Counter(broker.SubscribeMessageTotal, "endpoint", record.Topic, "status", "failure").Inc() // pc.kopts.Meter.Counter(broker.SubscribeMessageTotal, "endpoint", record.Topic, "status", "failure").Inc()
} // }
// pc.kopts.Meter.Counter(broker.SubscribeMessageInflight, "endpoint", record.Topic).Dec() // pc.kopts.Meter.Counter(broker.SubscribeMessageInflight, "endpoint", record.Topic).Dec()
if err == nil && pc.opts.AutoAck { if err == nil && pc.opts.AutoAck {
p.ack = true p.ack = true
@ -234,7 +241,7 @@ func (pc *consumer) consume() {
_ = eh(p) _ = eh(p)
} else { } else {
if pc.kopts.Logger.V(logger.ErrorLevel) { if pc.kopts.Logger.V(logger.ErrorLevel) {
pc.kopts.Logger.Errorf(pc.kopts.Context, "[kgo]: subscriber error: %v", err) pc.kopts.Logger.Error(pc.kopts.Context, "[kgo]: subscriber error", err)
} }
} }
} }
@ -246,7 +253,7 @@ func (pc *consumer) consume() {
pc.c.MarkCommitRecords(record) pc.c.MarkCommitRecords(record)
} else { } else {
eventPool.Put(p) eventPool.Put(p)
pc.kopts.Logger.Fatalf(pc.kopts.Context, "[kgo] ErrLostMessage wtf?") pc.kopts.Logger.Fatal(pc.kopts.Context, "[kgo] ErrLostMessage wtf?")
return return
} }
} }

View File

@ -12,9 +12,9 @@ import (
) )
type hookTracer struct { type hookTracer struct {
tracer tracer.Tracer
clientID string clientID string
group string group string
tracer tracer.Tracer
} }
var ( var (