Compare commits
10 Commits
Author | SHA1 | Date | |
---|---|---|---|
ff7a88fcce | |||
397f5414b3 | |||
9dd6efddf9 | |||
1e587b348a | |||
8475922d42 | |||
69fe6994ae | |||
6360feb351 | |||
0845f4873b | |||
747e35148f | |||
21f06fee6c |
34
broker.go
34
broker.go
@@ -12,8 +12,8 @@ import (
|
|||||||
|
|
||||||
type hookEvent struct {
|
type hookEvent struct {
|
||||||
log logger.Logger
|
log logger.Logger
|
||||||
fatalOnError bool
|
|
||||||
connected *atomic.Uint32
|
connected *atomic.Uint32
|
||||||
|
fatalOnError bool
|
||||||
}
|
}
|
||||||
|
|
||||||
var (
|
var (
|
||||||
@@ -27,43 +27,45 @@ var (
|
|||||||
|
|
||||||
func (m *hookEvent) OnGroupManageError(err error) {
|
func (m *hookEvent) OnGroupManageError(err error) {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
m.connected.Store(0)
|
// m.connected.Store(0)
|
||||||
if m.fatalOnError {
|
// if m.fatalOnError {
|
||||||
m.log.Fatal(context.TODO(), "kgo.OnGroupManageError", err)
|
m.log.Error(context.TODO(), "kgo.OnGroupManageError", err)
|
||||||
}
|
//}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m *hookEvent) OnBrokerConnect(_ kgo.BrokerMetadata, _ time.Duration, _ net.Conn, err error) {
|
func (m *hookEvent) OnBrokerConnect(_ kgo.BrokerMetadata, _ time.Duration, _ net.Conn, err error) {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
m.connected.Store(0)
|
// m.connected.Store(0)
|
||||||
if m.fatalOnError {
|
// if m.fatalOnError {
|
||||||
m.log.Fatal(context.TODO(), "kgo.OnBrokerConnect", err)
|
m.log.Error(context.TODO(), "kgo.OnBrokerConnect", err)
|
||||||
}
|
//}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m *hookEvent) OnBrokerDisconnect(_ kgo.BrokerMetadata, _ net.Conn) {
|
func (m *hookEvent) OnBrokerDisconnect(_ kgo.BrokerMetadata, _ net.Conn) {
|
||||||
m.connected.Store(0)
|
// m.connected.Store(0)
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m *hookEvent) OnBrokerWrite(_ kgo.BrokerMetadata, _ int16, _ int, _ time.Duration, _ time.Duration, err error) {
|
func (m *hookEvent) OnBrokerWrite(_ kgo.BrokerMetadata, _ int16, _ int, _ time.Duration, _ time.Duration, err error) {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
m.connected.Store(0)
|
// m.connected.Store(0)
|
||||||
if m.fatalOnError {
|
// if m.fatalOnError {
|
||||||
m.log.Fatal(context.TODO(), "kgo.OnBrokerWrite", err)
|
m.log.Error(context.TODO(), "kgo.OnBrokerWrite", err)
|
||||||
}
|
//}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m *hookEvent) OnBrokerRead(_ kgo.BrokerMetadata, _ int16, _ int, _ time.Duration, _ time.Duration, err error) {
|
func (m *hookEvent) OnBrokerRead(_ kgo.BrokerMetadata, _ int16, _ int, _ time.Duration, _ time.Duration, err error) {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
m.connected.Store(0)
|
// m.connected.Store(0)
|
||||||
|
m.log.Error(context.TODO(), "kgo.OnBrokerRead", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
func (m *hookEvent) OnProduceRecordUnbuffered(_ *kgo.Record, err error) {
|
func (m *hookEvent) OnProduceRecordUnbuffered(_ *kgo.Record, err error) {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
m.connected.Store(0)
|
// m.connected.Store(0)
|
||||||
|
m.log.Error(context.TODO(), "kgo.OnProduceRecordUnbuffered", err)
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
2
go.mod
2
go.mod
@@ -8,7 +8,7 @@ require (
|
|||||||
github.com/twmb/franz-go/pkg/kadm v1.15.0
|
github.com/twmb/franz-go/pkg/kadm v1.15.0
|
||||||
github.com/twmb/franz-go/pkg/kmsg v1.9.0
|
github.com/twmb/franz-go/pkg/kmsg v1.9.0
|
||||||
go.opentelemetry.io/otel v1.34.0
|
go.opentelemetry.io/otel v1.34.0
|
||||||
go.unistack.org/micro/v3 v3.11.41
|
go.unistack.org/micro/v3 v3.11.44
|
||||||
)
|
)
|
||||||
|
|
||||||
require (
|
require (
|
||||||
|
28
go.sum
28
go.sum
@@ -11,8 +11,6 @@ github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
|
|||||||
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
|
github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
|
||||||
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
|
||||||
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||||
github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc=
|
|
||||||
github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0=
|
|
||||||
github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
|
github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
|
||||||
github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
|
github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
|
||||||
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
|
github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
|
||||||
@@ -31,48 +29,30 @@ github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR
|
|||||||
github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o=
|
github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o=
|
||||||
github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
|
github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
|
||||||
github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
|
github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
|
||||||
github.com/twmb/franz-go v1.18.0 h1:25FjMZfdozBywVX+5xrWC2W+W76i0xykKjTdEeD2ejw=
|
|
||||||
github.com/twmb/franz-go v1.18.0/go.mod h1:zXCGy74M0p5FbXsLeASdyvfLFsBvTubVqctIaa5wQ+I=
|
|
||||||
github.com/twmb/franz-go v1.18.1 h1:D75xxCDyvTqBSiImFx2lkPduE39jz1vaD7+FNc+vMkc=
|
github.com/twmb/franz-go v1.18.1 h1:D75xxCDyvTqBSiImFx2lkPduE39jz1vaD7+FNc+vMkc=
|
||||||
github.com/twmb/franz-go v1.18.1/go.mod h1:Uzo77TarcLTUZeLuGq+9lNpSkfZI+JErv7YJhlDjs9M=
|
github.com/twmb/franz-go v1.18.1/go.mod h1:Uzo77TarcLTUZeLuGq+9lNpSkfZI+JErv7YJhlDjs9M=
|
||||||
github.com/twmb/franz-go/pkg/kadm v1.14.0 h1:nAn1co1lXzJQocpzyIyOFOjUBf4WHWs5/fTprXy2IZs=
|
|
||||||
github.com/twmb/franz-go/pkg/kadm v1.14.0/go.mod h1:XjOPz6ZaXXjrW2jVCfLuucP8H1w2TvD6y3PT2M+aAM4=
|
|
||||||
github.com/twmb/franz-go/pkg/kadm v1.15.0 h1:Yo3NAPfcsx3Gg9/hdhq4vmwO77TqRRkvpUcGWzjworc=
|
github.com/twmb/franz-go/pkg/kadm v1.15.0 h1:Yo3NAPfcsx3Gg9/hdhq4vmwO77TqRRkvpUcGWzjworc=
|
||||||
github.com/twmb/franz-go/pkg/kadm v1.15.0/go.mod h1:MUdcUtnf9ph4SFBLLA/XxE29rvLhWYLM9Ygb8dfSCvw=
|
github.com/twmb/franz-go/pkg/kadm v1.15.0/go.mod h1:MUdcUtnf9ph4SFBLLA/XxE29rvLhWYLM9Ygb8dfSCvw=
|
||||||
github.com/twmb/franz-go/pkg/kmsg v1.9.0 h1:JojYUph2TKAau6SBtErXpXGC7E3gg4vGZMv9xFU/B6M=
|
github.com/twmb/franz-go/pkg/kmsg v1.9.0 h1:JojYUph2TKAau6SBtErXpXGC7E3gg4vGZMv9xFU/B6M=
|
||||||
github.com/twmb/franz-go/pkg/kmsg v1.9.0/go.mod h1:CMbfazviCyY6HM0SXuG5t9vOwYDHRCSrJJyBAe5paqg=
|
github.com/twmb/franz-go/pkg/kmsg v1.9.0/go.mod h1:CMbfazviCyY6HM0SXuG5t9vOwYDHRCSrJJyBAe5paqg=
|
||||||
go.opentelemetry.io/otel v1.33.0 h1:/FerN9bax5LoK51X/sI0SVYrjSE0/yUL7DpxW4K3FWw=
|
|
||||||
go.opentelemetry.io/otel v1.33.0/go.mod h1:SUUkR6csvUQl+yjReHu5uM3EtVV7MBm5FHKRlNx4I8I=
|
|
||||||
go.opentelemetry.io/otel v1.34.0 h1:zRLXxLCgL1WyKsPVrgbSdMN4c0FMkDAskSTQP+0hdUY=
|
go.opentelemetry.io/otel v1.34.0 h1:zRLXxLCgL1WyKsPVrgbSdMN4c0FMkDAskSTQP+0hdUY=
|
||||||
go.opentelemetry.io/otel v1.34.0/go.mod h1:OWFPOQ+h4G8xpyjgqo4SxJYdDQ/qmRH+wivy7zzx9oI=
|
go.opentelemetry.io/otel v1.34.0/go.mod h1:OWFPOQ+h4G8xpyjgqo4SxJYdDQ/qmRH+wivy7zzx9oI=
|
||||||
go.unistack.org/micro-proto/v3 v3.4.1 h1:UTjLSRz2YZuaHk9iSlVqqsA50JQNAEK2ZFboGqtEa9Q=
|
go.unistack.org/micro-proto/v3 v3.4.1 h1:UTjLSRz2YZuaHk9iSlVqqsA50JQNAEK2ZFboGqtEa9Q=
|
||||||
go.unistack.org/micro-proto/v3 v3.4.1/go.mod h1:okx/cnOhzuCX0ggl/vToatbCupi0O44diiiLLsZ93Zo=
|
go.unistack.org/micro-proto/v3 v3.4.1/go.mod h1:okx/cnOhzuCX0ggl/vToatbCupi0O44diiiLLsZ93Zo=
|
||||||
go.unistack.org/micro/v3 v3.11.37 h1:ZcpnXAYEMcAwmnVb5b7o8/PylGnILxXMHaUlRrPmRI0=
|
go.unistack.org/micro/v3 v3.11.44 h1:A+T8zVcL2vlL66kn/Y4rqhtBybLO829wFEYZJYorDOU=
|
||||||
go.unistack.org/micro/v3 v3.11.37/go.mod h1:POGU5hstnAT9LH70m8FalyQSNi2GfIew71K75JenIZk=
|
go.unistack.org/micro/v3 v3.11.44/go.mod h1:13EFW2ps3BN9mpYbp9K0oQu/VDjEN6LJ4wwdom7hcXQ=
|
||||||
go.unistack.org/micro/v3 v3.11.41 h1:dP4sBLIZpMo+MWGe5bbESewK8wBzYm4Yik/67x4dEtQ=
|
|
||||||
go.unistack.org/micro/v3 v3.11.41/go.mod h1:POGU5hstnAT9LH70m8FalyQSNi2GfIew71K75JenIZk=
|
|
||||||
golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U=
|
|
||||||
golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
|
|
||||||
golang.org/x/crypto v0.35.0 h1:b15kiHdrGCHrP6LvwaQ3c03kgNhhiMgvlhxHQhmg2Xs=
|
golang.org/x/crypto v0.35.0 h1:b15kiHdrGCHrP6LvwaQ3c03kgNhhiMgvlhxHQhmg2Xs=
|
||||||
golang.org/x/crypto v0.35.0/go.mod h1:dy7dXNW32cAb/6/PRuTNsix8T+vJAqvuIy5Bli/x0YQ=
|
golang.org/x/crypto v0.35.0/go.mod h1:dy7dXNW32cAb/6/PRuTNsix8T+vJAqvuIy5Bli/x0YQ=
|
||||||
golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I=
|
golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I=
|
||||||
golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
|
golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
|
||||||
golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA=
|
|
||||||
golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
|
||||||
golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc=
|
golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc=
|
||||||
golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
|
||||||
golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo=
|
golang.org/x/text v0.22.0 h1:bofq7m3/HAFvbF51jz3Q9wLg3jkvSPuiZu/pD1XwgtM=
|
||||||
golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
|
golang.org/x/text v0.22.0/go.mod h1:YRoo4H8PVmsu+E3Ou7cqLVH8oXWIHVoX0jqUWALQhfY=
|
||||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20241216192217-9240e9c98484 h1:Z7FRVJPSMaHQxD0uXU8WdgFh8PseLM8Q8NzhnpMrBhQ=
|
|
||||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20241216192217-9240e9c98484/go.mod h1:lcTa1sDdWEIHMWlITnIczmw5w60CF9ffkb8Z+DVmmjA=
|
|
||||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20250224174004-546df14abb99 h1:ZSlhAUqC4r8TPzqLXQ0m3upBNZeF+Y8jQ3c4CR3Ujms=
|
google.golang.org/genproto/googleapis/rpc v0.0.0-20250224174004-546df14abb99 h1:ZSlhAUqC4r8TPzqLXQ0m3upBNZeF+Y8jQ3c4CR3Ujms=
|
||||||
google.golang.org/genproto/googleapis/rpc v0.0.0-20250224174004-546df14abb99/go.mod h1:LuRYeWDFV6WOn90g357N17oMCaxpgCnbi/44qJvDn2I=
|
google.golang.org/genproto/googleapis/rpc v0.0.0-20250224174004-546df14abb99/go.mod h1:LuRYeWDFV6WOn90g357N17oMCaxpgCnbi/44qJvDn2I=
|
||||||
google.golang.org/grpc v1.69.2 h1:U3S9QEtbXC0bYNvRtcoklF3xGtLViumSYxWykJS+7AU=
|
|
||||||
google.golang.org/grpc v1.69.2/go.mod h1:vyjdE6jLBI76dgpDojsFGNaHlxdjXN9ghpnd2o7JGZ4=
|
|
||||||
google.golang.org/grpc v1.70.0 h1:pWFv03aZoHzlRKHWicjsZytKAiYCtNS0dHbXnIdq7jQ=
|
google.golang.org/grpc v1.70.0 h1:pWFv03aZoHzlRKHWicjsZytKAiYCtNS0dHbXnIdq7jQ=
|
||||||
google.golang.org/grpc v1.70.0/go.mod h1:ofIJqVKDXx/JiXrwr2IG4/zwdH9txy3IlF40RmcJSQw=
|
google.golang.org/grpc v1.70.0/go.mod h1:ofIJqVKDXx/JiXrwr2IG4/zwdH9txy3IlF40RmcJSQw=
|
||||||
google.golang.org/protobuf v1.36.1 h1:yBPeRvTftaleIgM3PZ/WBIZ7XM/eEYAaEyCwvyjq/gk=
|
|
||||||
google.golang.org/protobuf v1.36.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE=
|
|
||||||
google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM=
|
google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM=
|
||||||
google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE=
|
google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE=
|
||||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||||
|
44
kgo.go
44
kgo.go
@@ -19,6 +19,7 @@ import (
|
|||||||
"go.unistack.org/micro/v3/metadata"
|
"go.unistack.org/micro/v3/metadata"
|
||||||
"go.unistack.org/micro/v3/semconv"
|
"go.unistack.org/micro/v3/semconv"
|
||||||
"go.unistack.org/micro/v3/tracer"
|
"go.unistack.org/micro/v3/tracer"
|
||||||
|
mjitter "go.unistack.org/micro/v3/util/jitter"
|
||||||
mrand "go.unistack.org/micro/v3/util/rand"
|
mrand "go.unistack.org/micro/v3/util/rand"
|
||||||
)
|
)
|
||||||
|
|
||||||
@@ -59,12 +60,14 @@ type Broker struct {
|
|||||||
c *kgo.Client
|
c *kgo.Client
|
||||||
connected *atomic.Uint32
|
connected *atomic.Uint32
|
||||||
|
|
||||||
|
done chan struct{}
|
||||||
|
|
||||||
kopts []kgo.Opt
|
kopts []kgo.Opt
|
||||||
subs []*Subscriber
|
subs []*Subscriber
|
||||||
|
|
||||||
opts broker.Options
|
opts broker.Options
|
||||||
|
|
||||||
sync.RWMutex
|
sync.RWMutex
|
||||||
|
|
||||||
init bool
|
init bool
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -143,6 +146,25 @@ func (k *Broker) connect(ctx context.Context, opts ...kgo.Opt) (*kgo.Client, *ho
|
|||||||
return nil, nil, err
|
return nil, nil, err
|
||||||
}
|
}
|
||||||
k.connected.Store(1)
|
k.connected.Store(1)
|
||||||
|
|
||||||
|
if fatalOnError {
|
||||||
|
go func() {
|
||||||
|
c := 3
|
||||||
|
n := 0
|
||||||
|
tc := mjitter.NewTicker(500*time.Millisecond, 1*time.Second)
|
||||||
|
defer tc.Stop()
|
||||||
|
for range tc.C {
|
||||||
|
if k.connected.Load() == 0 {
|
||||||
|
if n > c {
|
||||||
|
k.opts.Logger.Fatal(context.Background(), "broker fatal error")
|
||||||
|
}
|
||||||
|
n++
|
||||||
|
} else {
|
||||||
|
n = 0
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}()
|
||||||
|
}
|
||||||
return c, htracer, nil
|
return c, htracer, nil
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -204,6 +226,7 @@ func (k *Broker) Disconnect(ctx context.Context) error {
|
|||||||
}
|
}
|
||||||
|
|
||||||
k.connected.Store(0)
|
k.connected.Store(0)
|
||||||
|
close(k.done)
|
||||||
return nil
|
return nil
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -257,18 +280,6 @@ func (k *Broker) Publish(ctx context.Context, topic string, msg *broker.Message,
|
|||||||
}
|
}
|
||||||
|
|
||||||
func (k *Broker) publish(ctx context.Context, msgs []*broker.Message, opts ...broker.PublishOption) error {
|
func (k *Broker) publish(ctx context.Context, msgs []*broker.Message, opts ...broker.PublishOption) error {
|
||||||
k.Lock()
|
|
||||||
if k.connected.Load() == 0 {
|
|
||||||
c, _, err := k.connect(ctx, k.kopts...)
|
|
||||||
if err != nil {
|
|
||||||
k.Unlock()
|
|
||||||
return err
|
|
||||||
}
|
|
||||||
k.c = c
|
|
||||||
k.connected.Store(1)
|
|
||||||
}
|
|
||||||
k.Unlock()
|
|
||||||
|
|
||||||
options := broker.NewPublishOptions(opts...)
|
options := broker.NewPublishOptions(opts...)
|
||||||
records := make([]*kgo.Record, 0, len(msgs))
|
records := make([]*kgo.Record, 0, len(msgs))
|
||||||
var errs []string
|
var errs []string
|
||||||
@@ -391,12 +402,6 @@ func (k *Broker) Subscribe(ctx context.Context, topic string, handler broker.Han
|
|||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
if options.Context != nil {
|
|
||||||
if v, ok := options.Context.Value(fatalOnErrorKey{}).(bool); ok && v {
|
|
||||||
fatalOnError = v
|
|
||||||
}
|
|
||||||
}
|
|
||||||
|
|
||||||
sub := &Subscriber{
|
sub := &Subscriber{
|
||||||
topic: topic,
|
topic: topic,
|
||||||
opts: options,
|
opts: options,
|
||||||
@@ -492,5 +497,6 @@ func NewBroker(opts ...broker.Option) *Broker {
|
|||||||
connected: &atomic.Uint32{},
|
connected: &atomic.Uint32{},
|
||||||
opts: options,
|
opts: options,
|
||||||
kopts: kopts,
|
kopts: kopts,
|
||||||
|
done: make(chan struct{}),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
@@ -24,34 +24,44 @@ type tp struct {
|
|||||||
}
|
}
|
||||||
|
|
||||||
type consumer struct {
|
type consumer struct {
|
||||||
topic string
|
topic string
|
||||||
|
|
||||||
c *kgo.Client
|
c *kgo.Client
|
||||||
htracer *hookTracer
|
htracer *hookTracer
|
||||||
quit chan struct{}
|
|
||||||
done chan struct{}
|
|
||||||
recs chan kgo.FetchTopicPartition
|
|
||||||
kopts broker.Options
|
|
||||||
partition int32
|
|
||||||
opts broker.SubscribeOptions
|
|
||||||
handler broker.Handler
|
|
||||||
connected *atomic.Uint32
|
connected *atomic.Uint32
|
||||||
|
|
||||||
|
quit chan struct{}
|
||||||
|
done chan struct{}
|
||||||
|
recs chan kgo.FetchTopicPartition
|
||||||
|
|
||||||
|
handler broker.Handler
|
||||||
|
|
||||||
|
kopts broker.Options
|
||||||
|
opts broker.SubscribeOptions
|
||||||
|
|
||||||
|
partition int32
|
||||||
}
|
}
|
||||||
|
|
||||||
type Subscriber struct {
|
type Subscriber struct {
|
||||||
|
topic string
|
||||||
|
|
||||||
consumers map[tp]*consumer
|
consumers map[tp]*consumer
|
||||||
|
|
||||||
c *kgo.Client
|
c *kgo.Client
|
||||||
htracer *hookTracer
|
htracer *hookTracer
|
||||||
topic string
|
connected *atomic.Uint32
|
||||||
|
|
||||||
handler broker.Handler
|
handler broker.Handler
|
||||||
done chan struct{}
|
|
||||||
kopts broker.Options
|
|
||||||
opts broker.SubscribeOptions
|
|
||||||
|
|
||||||
connected *atomic.Uint32
|
done chan struct{}
|
||||||
sync.RWMutex
|
|
||||||
|
kopts broker.Options
|
||||||
|
opts broker.SubscribeOptions
|
||||||
|
|
||||||
closed bool
|
closed bool
|
||||||
fatalOnError bool
|
fatalOnError bool
|
||||||
|
|
||||||
|
sync.RWMutex
|
||||||
}
|
}
|
||||||
|
|
||||||
func (s *Subscriber) Client() *kgo.Client {
|
func (s *Subscriber) Client() *kgo.Client {
|
||||||
@@ -119,7 +129,7 @@ func (s *Subscriber) poll(ctx context.Context) {
|
|||||||
|
|
||||||
s.Lock()
|
s.Lock()
|
||||||
for p, l := range lmap {
|
for p, l := range lmap {
|
||||||
s.kopts.Meter.Counter(semconv.BrokerGroupLag, "topic", s.topic, "group", s.opts.Group, "partition", strconv.Itoa(int(p)), "lag", strconv.Itoa(int(l.Lag)))
|
s.kopts.Meter.Counter(semconv.BrokerGroupLag, "topic", s.topic, "group", s.opts.Group, "partition", strconv.Itoa(int(p))).Set(uint64(l.Lag))
|
||||||
}
|
}
|
||||||
s.Unlock()
|
s.Unlock()
|
||||||
|
|
||||||
@@ -160,7 +170,10 @@ func (s *Subscriber) killConsumers(ctx context.Context, lost map[string][]int32)
|
|||||||
for topic, partitions := range lost {
|
for topic, partitions := range lost {
|
||||||
for _, partition := range partitions {
|
for _, partition := range partitions {
|
||||||
tps := tp{topic, partition}
|
tps := tp{topic, partition}
|
||||||
pc := s.consumers[tps]
|
pc, ok := s.consumers[tps]
|
||||||
|
if !ok {
|
||||||
|
continue
|
||||||
|
}
|
||||||
delete(s.consumers, tps)
|
delete(s.consumers, tps)
|
||||||
close(pc.quit)
|
close(pc.quit)
|
||||||
if s.kopts.Logger.V(logger.DebugLevel) {
|
if s.kopts.Logger.V(logger.DebugLevel) {
|
||||||
@@ -174,7 +187,7 @@ func (s *Subscriber) killConsumers(ctx context.Context, lost map[string][]int32)
|
|||||||
|
|
||||||
func (s *Subscriber) autocommit(_ *kgo.Client, _ *kmsg.OffsetCommitRequest, _ *kmsg.OffsetCommitResponse, err error) {
|
func (s *Subscriber) autocommit(_ *kgo.Client, _ *kmsg.OffsetCommitRequest, _ *kmsg.OffsetCommitResponse, err error) {
|
||||||
if err != nil {
|
if err != nil {
|
||||||
s.connected.Store(0)
|
// s.connected.Store(0)
|
||||||
if s.fatalOnError {
|
if s.fatalOnError {
|
||||||
s.kopts.Logger.Fatal(context.TODO(), "kgo.AutoCommitCallback error", err)
|
s.kopts.Logger.Fatal(context.TODO(), "kgo.AutoCommitCallback error", err)
|
||||||
}
|
}
|
||||||
@@ -261,11 +274,17 @@ func (pc *consumer) consume() {
|
|||||||
} else if pc.opts.BodyOnly {
|
} else if pc.opts.BodyOnly {
|
||||||
p.msg.Body = record.Value
|
p.msg.Body = record.Value
|
||||||
} else {
|
} else {
|
||||||
sp.AddEvent("codec unmarshal start")
|
if sp != nil {
|
||||||
|
sp.AddEvent("codec unmarshal start")
|
||||||
|
}
|
||||||
err := pc.kopts.Codec.Unmarshal(record.Value, p.msg)
|
err := pc.kopts.Codec.Unmarshal(record.Value, p.msg)
|
||||||
sp.AddEvent("codec unmarshal stop")
|
if sp != nil {
|
||||||
|
sp.AddEvent("codec unmarshal stop")
|
||||||
|
}
|
||||||
if err != nil {
|
if err != nil {
|
||||||
sp.SetStatus(tracer.SpanStatusError, err.Error())
|
if sp != nil {
|
||||||
|
sp.SetStatus(tracer.SpanStatusError, err.Error())
|
||||||
|
}
|
||||||
pc.kopts.Meter.Counter(semconv.SubscribeMessageTotal, "endpoint", record.Topic, "topic", record.Topic, "status", "failure").Inc()
|
pc.kopts.Meter.Counter(semconv.SubscribeMessageTotal, "endpoint", record.Topic, "topic", record.Topic, "status", "failure").Inc()
|
||||||
p.err = err
|
p.err = err
|
||||||
p.msg.Body = record.Value
|
p.msg.Body = record.Value
|
||||||
@@ -276,7 +295,7 @@ func (pc *consumer) consume() {
|
|||||||
pc.c.MarkCommitRecords(record)
|
pc.c.MarkCommitRecords(record)
|
||||||
} else {
|
} else {
|
||||||
eventPool.Put(p)
|
eventPool.Put(p)
|
||||||
pc.connected.Store(0)
|
// pc.connected.Store(0)
|
||||||
pc.kopts.Logger.Fatal(pc.kopts.Context, "[kgo] ErrLostMessage wtf?")
|
pc.kopts.Logger.Fatal(pc.kopts.Context, "[kgo] ErrLostMessage wtf?")
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
@@ -293,19 +312,27 @@ func (pc *consumer) consume() {
|
|||||||
pc.kopts.Meter.Summary(semconv.SubscribeMessageLatencyMicroseconds, "endpoint", record.Topic, "topic", record.Topic).Update(te.Seconds())
|
pc.kopts.Meter.Summary(semconv.SubscribeMessageLatencyMicroseconds, "endpoint", record.Topic, "topic", record.Topic).Update(te.Seconds())
|
||||||
pc.kopts.Meter.Histogram(semconv.SubscribeMessageDurationSeconds, "endpoint", record.Topic, "topic", record.Topic).Update(te.Seconds())
|
pc.kopts.Meter.Histogram(semconv.SubscribeMessageDurationSeconds, "endpoint", record.Topic, "topic", record.Topic).Update(te.Seconds())
|
||||||
eventPool.Put(p)
|
eventPool.Put(p)
|
||||||
pc.connected.Store(0)
|
// pc.connected.Store(0)
|
||||||
pc.kopts.Logger.Fatal(pc.kopts.Context, "[kgo] Unmarshal err not handled wtf?")
|
pc.kopts.Logger.Fatal(pc.kopts.Context, "[kgo] Unmarshal err not handled wtf?")
|
||||||
sp.Finish()
|
if sp != nil {
|
||||||
|
sp.Finish()
|
||||||
|
}
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
sp.AddEvent("handler start")
|
if sp != nil {
|
||||||
|
sp.AddEvent("handler start")
|
||||||
|
}
|
||||||
err := pc.handler(p)
|
err := pc.handler(p)
|
||||||
sp.AddEvent("handler stop")
|
if sp != nil {
|
||||||
|
sp.AddEvent("handler stop")
|
||||||
|
}
|
||||||
if err == nil {
|
if err == nil {
|
||||||
pc.kopts.Meter.Counter(semconv.SubscribeMessageTotal, "endpoint", record.Topic, "topic", record.Topic, "status", "success").Inc()
|
pc.kopts.Meter.Counter(semconv.SubscribeMessageTotal, "endpoint", record.Topic, "topic", record.Topic, "status", "success").Inc()
|
||||||
} else {
|
} else {
|
||||||
sp.SetStatus(tracer.SpanStatusError, err.Error())
|
if sp != nil {
|
||||||
|
sp.SetStatus(tracer.SpanStatusError, err.Error())
|
||||||
|
}
|
||||||
pc.kopts.Meter.Counter(semconv.SubscribeMessageTotal, "endpoint", record.Topic, "topic", record.Topic, "status", "failure").Inc()
|
pc.kopts.Meter.Counter(semconv.SubscribeMessageTotal, "endpoint", record.Topic, "topic", record.Topic, "status", "failure").Inc()
|
||||||
}
|
}
|
||||||
pc.kopts.Meter.Counter(semconv.SubscribeMessageInflight, "endpoint", record.Topic, "topic", record.Topic).Dec()
|
pc.kopts.Meter.Counter(semconv.SubscribeMessageInflight, "endpoint", record.Topic, "topic", record.Topic).Dec()
|
||||||
@@ -314,9 +341,13 @@ func (pc *consumer) consume() {
|
|||||||
} else if err != nil {
|
} else if err != nil {
|
||||||
p.err = err
|
p.err = err
|
||||||
if eh != nil {
|
if eh != nil {
|
||||||
sp.AddEvent("error handler start")
|
if sp != nil {
|
||||||
|
sp.AddEvent("error handler start")
|
||||||
|
}
|
||||||
_ = eh(p)
|
_ = eh(p)
|
||||||
sp.AddEvent("error handler stop")
|
if sp != nil {
|
||||||
|
sp.AddEvent("error handler stop")
|
||||||
|
}
|
||||||
} else {
|
} else {
|
||||||
if pc.kopts.Logger.V(logger.ErrorLevel) {
|
if pc.kopts.Logger.V(logger.ErrorLevel) {
|
||||||
pc.kopts.Logger.Error(pc.kopts.Context, "[kgo]: subscriber error", err)
|
pc.kopts.Logger.Error(pc.kopts.Context, "[kgo]: subscriber error", err)
|
||||||
@@ -331,13 +362,17 @@ func (pc *consumer) consume() {
|
|||||||
pc.c.MarkCommitRecords(record)
|
pc.c.MarkCommitRecords(record)
|
||||||
} else {
|
} else {
|
||||||
eventPool.Put(p)
|
eventPool.Put(p)
|
||||||
pc.connected.Store(0)
|
// pc.connected.Store(0)
|
||||||
pc.kopts.Logger.Fatal(pc.kopts.Context, "[kgo] ErrLostMessage wtf?")
|
pc.kopts.Logger.Fatal(pc.kopts.Context, "[kgo] ErrLostMessage wtf?")
|
||||||
sp.SetStatus(tracer.SpanStatusError, "ErrLostMessage")
|
if sp != nil {
|
||||||
sp.Finish()
|
sp.SetStatus(tracer.SpanStatusError, "ErrLostMessage")
|
||||||
|
sp.Finish()
|
||||||
|
}
|
||||||
return
|
return
|
||||||
}
|
}
|
||||||
sp.Finish()
|
if sp != nil {
|
||||||
|
sp.Finish()
|
||||||
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
19
tracer.go
19
tracer.go
@@ -32,6 +32,9 @@ var (
|
|||||||
// the record's context, so it can be ended in the OnProduceRecordUnbuffered
|
// the record's context, so it can be ended in the OnProduceRecordUnbuffered
|
||||||
// hook.
|
// hook.
|
||||||
func (m *hookTracer) OnProduceRecordBuffered(r *kgo.Record) {
|
func (m *hookTracer) OnProduceRecordBuffered(r *kgo.Record) {
|
||||||
|
if !m.tracer.Enabled() {
|
||||||
|
return
|
||||||
|
}
|
||||||
// Set up span options.
|
// Set up span options.
|
||||||
attrs := []interface{}{
|
attrs := []interface{}{
|
||||||
messagingSystem,
|
messagingSystem,
|
||||||
@@ -77,6 +80,9 @@ func (m *hookTracer) OnProduceRecordBuffered(r *kgo.Record) {
|
|||||||
// It sets attributes with values unset when producing and records any error
|
// It sets attributes with values unset when producing and records any error
|
||||||
// that occurred during the publish operation.
|
// that occurred during the publish operation.
|
||||||
func (m *hookTracer) OnProduceRecordUnbuffered(r *kgo.Record, err error) {
|
func (m *hookTracer) OnProduceRecordUnbuffered(r *kgo.Record, err error) {
|
||||||
|
if !m.tracer.Enabled() {
|
||||||
|
return
|
||||||
|
}
|
||||||
if span, ok := tracer.SpanFromContext(r.Context); ok {
|
if span, ok := tracer.SpanFromContext(r.Context); ok {
|
||||||
span.AddLabels(
|
span.AddLabels(
|
||||||
semconv.MessagingKafkaDestinationPartition(int(r.Partition)),
|
semconv.MessagingKafkaDestinationPartition(int(r.Partition)),
|
||||||
@@ -96,6 +102,9 @@ func (m *hookTracer) OnProduceRecordUnbuffered(r *kgo.Record, err error) {
|
|||||||
// OnFetchRecordUnbuffered hook and can be used in downstream consumer
|
// OnFetchRecordUnbuffered hook and can be used in downstream consumer
|
||||||
// processing.
|
// processing.
|
||||||
func (m *hookTracer) OnFetchRecordBuffered(r *kgo.Record) {
|
func (m *hookTracer) OnFetchRecordBuffered(r *kgo.Record) {
|
||||||
|
if !m.tracer.Enabled() {
|
||||||
|
return
|
||||||
|
}
|
||||||
// Set up the span options.
|
// Set up the span options.
|
||||||
attrs := []interface{}{
|
attrs := []interface{}{
|
||||||
messagingSystem,
|
messagingSystem,
|
||||||
@@ -141,6 +150,9 @@ func (m *hookTracer) OnFetchRecordBuffered(r *kgo.Record) {
|
|||||||
// OnFetchRecordUnbuffered continues and ends the "receive" span for an
|
// OnFetchRecordUnbuffered continues and ends the "receive" span for an
|
||||||
// unbuffered record.
|
// unbuffered record.
|
||||||
func (m *hookTracer) OnFetchRecordUnbuffered(r *kgo.Record, _ bool) {
|
func (m *hookTracer) OnFetchRecordUnbuffered(r *kgo.Record, _ bool) {
|
||||||
|
if !m.tracer.Enabled() {
|
||||||
|
return
|
||||||
|
}
|
||||||
span, _ := tracer.SpanFromContext(r.Context)
|
span, _ := tracer.SpanFromContext(r.Context)
|
||||||
span.Finish()
|
span.Finish()
|
||||||
}
|
}
|
||||||
@@ -155,6 +167,13 @@ func (m *hookTracer) OnFetchRecordUnbuffered(r *kgo.Record, _ bool) {
|
|||||||
// not a record which has been created for producing, so call this at the start of each
|
// not a record which has been created for producing, so call this at the start of each
|
||||||
// iteration of your processing for the record.
|
// iteration of your processing for the record.
|
||||||
func (m *hookTracer) WithProcessSpan(r *kgo.Record) (context.Context, tracer.Span) {
|
func (m *hookTracer) WithProcessSpan(r *kgo.Record) (context.Context, tracer.Span) {
|
||||||
|
if r.Context == nil {
|
||||||
|
r.Context = context.Background()
|
||||||
|
}
|
||||||
|
|
||||||
|
if !m.tracer.Enabled() {
|
||||||
|
return r.Context, nil
|
||||||
|
}
|
||||||
// Set up the span options.
|
// Set up the span options.
|
||||||
attrs := []interface{}{
|
attrs := []interface{}{
|
||||||
messagingSystem,
|
messagingSystem,
|
||||||
|
Reference in New Issue
Block a user