remove noisy logs from event hooks and add tests (#192)

* update ci

* attempt to fix lint/test job

* attempt to fix lint/test job

.github/workflows/job_coverage.yml

@@ -0,0 +1,53 @@
+name: coverage
+
+on:
+  push:
+    branches: [ main, v3, v4 ]
+    paths-ignore:
+      - '.github/**'
+      - '.gitea/**'
+  pull_request:
+    branches: [ main, v3, v4 ]
+
+jobs:
+
+  build:
+    if: github.server_url != 'https://github.com'
+    runs-on: ubuntu-latest
+    steps:
+    - name: checkout code
+      uses: actions/checkout@v4
+      with:
+        filter: 'blob:none'
+
+    - name: setup go
+      uses: actions/setup-go@v5
+      with:
+        cache-dependency-path: "**/*.sum"
+        go-version: 'stable'
+
+    - name: test coverage
+      run: |
+        go test -v -cover ./... -covermode=count -coverprofile coverage.out -coverpkg ./...
+        go tool cover -func coverage.out -o coverage.out
+
+    - name: coverage badge
+      uses: tj-actions/coverage-badge-go@v2
+      with:
+        green: 80
+        filename: coverage.out
+
+    - uses: stefanzweifel/git-auto-commit-action@v4
+      name: autocommit
+      with:
+        commit_message: Apply Code Coverage Badge
+        skip_fetch: false
+        skip_checkout: false
+        file_pattern: ./README.md
+
+    - name: push
+      if: steps.auto-commit-action.outputs.changes_detected == 'true'
+      uses: ad-m/github-push-action@master
+      with:
+        github_token: ${{ github.token }}
+        branch: ${{ github.ref }}

.github/workflows/job_lint.yml

@@ -3,10 +3,10 @@ name: lint
 on:
   pull_request:
     types: [opened, reopened, synchronize]
-    branches:
+    branches: [ master, v3, v4 ]
-    - master
+    paths-ignore:
-    - v3
+      - '.github/**'
-    - v4
+      - '.gitea/**'
 
 jobs:
   lint:
@@ -24,6 +24,6 @@ jobs:
     - name: setup deps
       run: go get -v ./...
     - name: run lint
-      uses: https://github.com/golangci/golangci-lint-action@v6
+      uses: golangci/golangci-lint-action@v6
       with:
         version: 'latest'

.github/workflows/job_test.yml

@@ -3,15 +3,12 @@ name: test
 on:
   pull_request:
     types: [opened, reopened, synchronize]
-    branches:
+    branches: [ master, v3, v4 ]
-    - master
-    - v3
-    - v4
   push:
-    branches:
+    branches: [ master, v3, v4 ]
-    - master
+    paths-ignore:
-    - v3
+      - '.github/**'
-    - v4
+      - '.gitea/**'
 
 jobs:
   test:

.github/workflows/job_tests.yml

@@ -3,15 +3,12 @@ name: test
 on:
   pull_request:
     types: [opened, reopened, synchronize]
-    branches:
+    branches: [ master, v3, v4 ]
-    - master
-    - v3
-    - v4
   push:
-    branches:
+    branches: [ master, v3, v4 ]
-    - master
+    paths-ignore:
-    - v3
+      - '.github/**'
-    - v4
+      - '.gitea/**'
 
 jobs:
   test:
@@ -35,19 +32,19 @@ jobs:
         go-version: 'stable'
     - name: setup go work
       env:
-        GOWORK: /workspace/${{ github.repository_owner }}/go.work
+        GOWORK: ${{ github.workspace }}/go.work
       run: |
         go work init
         go work use .
         go work use micro-tests
     - name: setup deps
       env:
-        GOWORK: /workspace/${{ github.repository_owner }}/go.work
+        GOWORK: ${{ github.workspace }}/go.work
       run: go get -v ./...
     - name: run tests
       env:
         INTEGRATION_TESTS: yes
-        GOWORK: /workspace/${{ github.repository_owner }}/go.work
+        GOWORK: ${{ github.workspace }}/go.work
       run: |
         cd micro-tests
         go test -mod readonly -v ./... || true

.golangci.yml

@@ -1,21 +1,5 @@
 run:
   concurrency: 8
-  deadline: 5m
+  timeout: 5m
   issues-exit-code: 1
   tests: true
-
-linters:
-  enable:
-    - staticcheck
-    - unused
-    - gosimple
-    - govet
-    - prealloc
-    - unconvert
-    - nakedret
-
-linters-settings:
-  govet:
-    check-all: true
-    enable:
-      - fieldalignment

errors.go

@@ -0,0 +1,10 @@
+package kgo
+
+import (
+	"context"
+	"errors"
+)
+
+func isContextError(err error) bool {
+	return errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded)
+}

go.mod

@@ -1,25 +1,30 @@
 module go.unistack.org/micro-broker-kgo/v3
 
-go 1.23.0
+go 1.23.8
+
+toolchain go1.24.3
 
 require (
 	github.com/google/uuid v1.6.0
-	github.com/twmb/franz-go v1.18.1
+	github.com/stretchr/testify v1.10.0
-	github.com/twmb/franz-go/pkg/kadm v1.15.0
+	github.com/twmb/franz-go v1.19.5
-	github.com/twmb/franz-go/pkg/kmsg v1.9.0
+	github.com/twmb/franz-go/pkg/kadm v1.16.0
+	github.com/twmb/franz-go/pkg/kmsg v1.11.2
 	go.opentelemetry.io/otel v1.34.0
-	go.unistack.org/micro/v3 v3.11.41
+	go.unistack.org/micro/v3 v3.11.44
 )
 
 require (
 	github.com/ash3in/uuidv8 v1.2.0 // indirect
+	github.com/davecgh/go-spew v1.1.2-0.20180830191138-d8f796af33cc // indirect
 	github.com/klauspost/compress v1.18.0 // indirect
 	github.com/kr/pretty v0.3.1 // indirect
 	github.com/matoous/go-nanoid v1.5.1 // indirect
 	github.com/pierrec/lz4/v4 v4.1.22 // indirect
+	github.com/pmezard/go-difflib v1.0.1-0.20181226105442-5d4384ee4fb2 // indirect
 	go.unistack.org/micro-proto/v3 v3.4.1 // indirect
-	golang.org/x/crypto v0.35.0 // indirect
+	golang.org/x/crypto v0.38.0 // indirect
-	golang.org/x/sys v0.30.0 // indirect
+	golang.org/x/sys v0.33.0 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20250224174004-546df14abb99 // indirect
 	google.golang.org/grpc v1.70.0 // indirect
 	google.golang.org/protobuf v1.36.5 // indirect

go.sum

@@ -11,8 +11,6 @@ github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI=
 github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY=
 github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
 github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/klauspost/compress v1.17.11 h1:In6xLpyWOi1+C7tXUUWv2ot1QvBjxevKAaI6IXrJmUc=
-github.com/klauspost/compress v1.17.11/go.mod h1:pMDklpSncoRMuLFrf1W9Ss9KT+0rH90U12bZKk7uwG0=
 github.com/klauspost/compress v1.18.0 h1:c/Cqfb0r+Yi+JtIEq73FWXVkRonBlf0CRNYc8Zttxdo=
 github.com/klauspost/compress v1.18.0/go.mod h1:2Pp+KzxcywXVXMr50+X0Q/Lsb43OQHYWRCY2AiWywWQ=
 github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE=
@@ -31,48 +29,30 @@ github.com/rogpeppe/go-internal v1.13.1 h1:KvO1DLK/DRN07sQ1LQKScxyZJuNnedQ5/wKSR
 github.com/rogpeppe/go-internal v1.13.1/go.mod h1:uMEvuHeurkdAXX61udpOXGD/AzZDWNMNyH2VO9fmH0o=
 github.com/stretchr/testify v1.10.0 h1:Xv5erBjTwe/5IxqUQTdXv5kgmIvbHo3QQyRwhJsOfJA=
 github.com/stretchr/testify v1.10.0/go.mod h1:r2ic/lqez/lEtzL7wO/rwa5dbSLXVDPFyf8C91i36aY=
-github.com/twmb/franz-go v1.18.0 h1:25FjMZfdozBywVX+5xrWC2W+W76i0xykKjTdEeD2ejw=
+github.com/twmb/franz-go v1.19.5 h1:W7+o8D0RsQsedqib71OVlLeZ0zI6CbFra7yTYhZTs5Y=
-github.com/twmb/franz-go v1.18.0/go.mod h1:zXCGy74M0p5FbXsLeASdyvfLFsBvTubVqctIaa5wQ+I=
+github.com/twmb/franz-go v1.19.5/go.mod h1:4kFJ5tmbbl7asgwAGVuyG1ZMx0NNpYk7EqflvWfPCpM=
-github.com/twmb/franz-go v1.18.1 h1:D75xxCDyvTqBSiImFx2lkPduE39jz1vaD7+FNc+vMkc=
+github.com/twmb/franz-go/pkg/kadm v1.16.0 h1:STMs1t5lYR5mR974PSiwNzE5TvsosByTp+rKXLOhAjE=
-github.com/twmb/franz-go v1.18.1/go.mod h1:Uzo77TarcLTUZeLuGq+9lNpSkfZI+JErv7YJhlDjs9M=
+github.com/twmb/franz-go/pkg/kadm v1.16.0/go.mod h1:MUdcUtnf9ph4SFBLLA/XxE29rvLhWYLM9Ygb8dfSCvw=
-github.com/twmb/franz-go/pkg/kadm v1.14.0 h1:nAn1co1lXzJQocpzyIyOFOjUBf4WHWs5/fTprXy2IZs=
+github.com/twmb/franz-go/pkg/kmsg v1.11.2 h1:hIw75FpwcAjgeyfIGFqivAvwC5uNIOWRGvQgZhH4mhg=
-github.com/twmb/franz-go/pkg/kadm v1.14.0/go.mod h1:XjOPz6ZaXXjrW2jVCfLuucP8H1w2TvD6y3PT2M+aAM4=
+github.com/twmb/franz-go/pkg/kmsg v1.11.2/go.mod h1:CFfkkLysDNmukPYhGzuUcDtf46gQSqCZHMW1T4Z+wDE=
-github.com/twmb/franz-go/pkg/kadm v1.15.0 h1:Yo3NAPfcsx3Gg9/hdhq4vmwO77TqRRkvpUcGWzjworc=
-github.com/twmb/franz-go/pkg/kadm v1.15.0/go.mod h1:MUdcUtnf9ph4SFBLLA/XxE29rvLhWYLM9Ygb8dfSCvw=
-github.com/twmb/franz-go/pkg/kmsg v1.9.0 h1:JojYUph2TKAau6SBtErXpXGC7E3gg4vGZMv9xFU/B6M=
-github.com/twmb/franz-go/pkg/kmsg v1.9.0/go.mod h1:CMbfazviCyY6HM0SXuG5t9vOwYDHRCSrJJyBAe5paqg=
-go.opentelemetry.io/otel v1.33.0 h1:/FerN9bax5LoK51X/sI0SVYrjSE0/yUL7DpxW4K3FWw=
-go.opentelemetry.io/otel v1.33.0/go.mod h1:SUUkR6csvUQl+yjReHu5uM3EtVV7MBm5FHKRlNx4I8I=
 go.opentelemetry.io/otel v1.34.0 h1:zRLXxLCgL1WyKsPVrgbSdMN4c0FMkDAskSTQP+0hdUY=
 go.opentelemetry.io/otel v1.34.0/go.mod h1:OWFPOQ+h4G8xpyjgqo4SxJYdDQ/qmRH+wivy7zzx9oI=
 go.unistack.org/micro-proto/v3 v3.4.1 h1:UTjLSRz2YZuaHk9iSlVqqsA50JQNAEK2ZFboGqtEa9Q=
 go.unistack.org/micro-proto/v3 v3.4.1/go.mod h1:okx/cnOhzuCX0ggl/vToatbCupi0O44diiiLLsZ93Zo=
-go.unistack.org/micro/v3 v3.11.37 h1:ZcpnXAYEMcAwmnVb5b7o8/PylGnILxXMHaUlRrPmRI0=
+go.unistack.org/micro/v3 v3.11.44 h1:A+T8zVcL2vlL66kn/Y4rqhtBybLO829wFEYZJYorDOU=
-go.unistack.org/micro/v3 v3.11.37/go.mod h1:POGU5hstnAT9LH70m8FalyQSNi2GfIew71K75JenIZk=
+go.unistack.org/micro/v3 v3.11.44/go.mod h1:13EFW2ps3BN9mpYbp9K0oQu/VDjEN6LJ4wwdom7hcXQ=
-go.unistack.org/micro/v3 v3.11.41 h1:dP4sBLIZpMo+MWGe5bbESewK8wBzYm4Yik/67x4dEtQ=
+golang.org/x/crypto v0.38.0 h1:jt+WWG8IZlBnVbomuhg2Mdq0+BBQaHbtqHEFEigjUV8=
-go.unistack.org/micro/v3 v3.11.41/go.mod h1:POGU5hstnAT9LH70m8FalyQSNi2GfIew71K75JenIZk=
+golang.org/x/crypto v0.38.0/go.mod h1:MvrbAqul58NNYPKnOra203SB9vpuZW0e+RRZV+Ggqjw=
-golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U=
-golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk=
-golang.org/x/crypto v0.35.0 h1:b15kiHdrGCHrP6LvwaQ3c03kgNhhiMgvlhxHQhmg2Xs=
-golang.org/x/crypto v0.35.0/go.mod h1:dy7dXNW32cAb/6/PRuTNsix8T+vJAqvuIy5Bli/x0YQ=
 golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I=
 golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4=
-golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA=
+golang.org/x/sys v0.33.0 h1:q3i8TbbEz+JRD9ywIRlyRAQbM0qF7hu24q3teo2hbuw=
-golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.33.0/go.mod h1:BJP2sWEmIv4KK5OTEluFJCKSidICx8ciO85XgH3Ak8k=
-golang.org/x/sys v0.30.0 h1:QjkSwP/36a20jFYWkSue1YwXzLmsV5Gfq7Eiy72C1uc=
+golang.org/x/text v0.25.0 h1:qVyWApTSYLk/drJRO5mDlNYskwQznZmkpV2c8q9zls4=
-golang.org/x/sys v0.30.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/text v0.25.0/go.mod h1:WEdwpYrmk1qmdHvhkSTNPm3app7v4rsT8F2UD6+VHIA=
-golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo=
-golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20241216192217-9240e9c98484 h1:Z7FRVJPSMaHQxD0uXU8WdgFh8PseLM8Q8NzhnpMrBhQ=
-google.golang.org/genproto/googleapis/rpc v0.0.0-20241216192217-9240e9c98484/go.mod h1:lcTa1sDdWEIHMWlITnIczmw5w60CF9ffkb8Z+DVmmjA=
 google.golang.org/genproto/googleapis/rpc v0.0.0-20250224174004-546df14abb99 h1:ZSlhAUqC4r8TPzqLXQ0m3upBNZeF+Y8jQ3c4CR3Ujms=
 google.golang.org/genproto/googleapis/rpc v0.0.0-20250224174004-546df14abb99/go.mod h1:LuRYeWDFV6WOn90g357N17oMCaxpgCnbi/44qJvDn2I=
-google.golang.org/grpc v1.69.2 h1:U3S9QEtbXC0bYNvRtcoklF3xGtLViumSYxWykJS+7AU=
-google.golang.org/grpc v1.69.2/go.mod h1:vyjdE6jLBI76dgpDojsFGNaHlxdjXN9ghpnd2o7JGZ4=
 google.golang.org/grpc v1.70.0 h1:pWFv03aZoHzlRKHWicjsZytKAiYCtNS0dHbXnIdq7jQ=
 google.golang.org/grpc v1.70.0/go.mod h1:ofIJqVKDXx/JiXrwr2IG4/zwdH9txy3IlF40RmcJSQw=
-google.golang.org/protobuf v1.36.1 h1:yBPeRvTftaleIgM3PZ/WBIZ7XM/eEYAaEyCwvyjq/gk=
-google.golang.org/protobuf v1.36.1/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE=
 google.golang.org/protobuf v1.36.5 h1:tPhr+woSbjfYvY6/GPufUoYizxw1cF/yFoxJ2fmpwlM=
 google.golang.org/protobuf v1.36.5/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=

hook_event.go

@@ -12,8 +12,8 @@ import (
 
 type hookEvent struct {
 	log          logger.Logger
-	fatalOnError bool
 	connected    *atomic.Uint32
+	fatalOnError bool
 }
 
 var (
@@ -26,44 +26,48 @@ var (
 )
 
 func (m *hookEvent) OnGroupManageError(err error) {
-	if err != nil {
+	switch {
-		m.connected.Store(0)
+	case err == nil || isContextError(err) || kgo.IsRetryableBrokerErr(err):
-		if m.fatalOnError {
+		return
-			m.log.Fatal(context.TODO(), "kgo.OnGroupManageError", err)
+	default:
-		}
+		m.log.Error(context.TODO(), "kgo.OnGroupManageError", err)
 	}
 }
 
 func (m *hookEvent) OnBrokerConnect(_ kgo.BrokerMetadata, _ time.Duration, _ net.Conn, err error) {
-	if err != nil {
+	switch {
-		m.connected.Store(0)
+	case err == nil || isContextError(err) || kgo.IsRetryableBrokerErr(err):
-		if m.fatalOnError {
+		return
-			m.log.Fatal(context.TODO(), "kgo.OnBrokerConnect", err)
+	default:
-		}
+		m.log.Error(context.TODO(), "kgo.OnBrokerConnect", err)
 	}
 }
 
-func (m *hookEvent) OnBrokerDisconnect(_ kgo.BrokerMetadata, _ net.Conn) {
+func (m *hookEvent) OnBrokerDisconnect(_ kgo.BrokerMetadata, _ net.Conn) {}
-	m.connected.Store(0)
-}
 
 func (m *hookEvent) OnBrokerWrite(_ kgo.BrokerMetadata, _ int16, _ int, _ time.Duration, _ time.Duration, err error) {
-	if err != nil {
+	switch {
-		m.connected.Store(0)
+	case err == nil || isContextError(err) || kgo.IsRetryableBrokerErr(err):
-		if m.fatalOnError {
+		return
-			m.log.Fatal(context.TODO(), "kgo.OnBrokerWrite", err)
+	default:
-		}
+		m.log.Error(context.TODO(), "kgo.OnBrokerWrite", err)
 	}
 }
 
 func (m *hookEvent) OnBrokerRead(_ kgo.BrokerMetadata, _ int16, _ int, _ time.Duration, _ time.Duration, err error) {
-	if err != nil {
+	switch {
-		m.connected.Store(0)
+	case err == nil || isContextError(err) || kgo.IsRetryableBrokerErr(err):
+		return
+	default:
+		m.log.Error(context.TODO(), "kgo.OnBrokerRead", err)
 	}
 }
 
 func (m *hookEvent) OnProduceRecordUnbuffered(_ *kgo.Record, err error) {
-	if err != nil {
+	switch {
-		m.connected.Store(0)
+	case err == nil || isContextError(err) || kgo.IsRetryableBrokerErr(err):
+		return
+	default:
+		m.log.Error(context.TODO(), "kgo.OnProduceRecordUnbuffered", err)
 	}
 }

hook_event_test.go

@@ -0,0 +1,362 @@
+package kgo
+
+import (
+	"context"
+	"errors"
+	"io"
+	"net"
+	"os"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	"github.com/twmb/franz-go/pkg/kgo"
+	"go.unistack.org/micro/v3/logger"
+)
+
+func TestHookEvent_OnGroupManageError(t *testing.T) {
+	tests := []struct {
+		name                  string
+		inputErr              error
+		expectedErrorIsCalled bool
+		expectedErrorMsg      string
+	}{
+		{
+			name:                  "error is nil",
+			inputErr:              nil,
+			expectedErrorIsCalled: false,
+		},
+		{
+			name:                  "context canceled",
+			inputErr:              context.Canceled,
+			expectedErrorIsCalled: false,
+		},
+		{
+			name:                  "context deadline exceeded",
+			inputErr:              context.DeadlineExceeded,
+			expectedErrorIsCalled: false,
+		},
+		{
+			name:                  "retryable error: deadline exceeded (os package)",
+			inputErr:              os.ErrDeadlineExceeded,
+			expectedErrorIsCalled: false,
+		},
+		{
+			name:                  "retryable error: EOF (io package)",
+			inputErr:              io.EOF,
+			expectedErrorIsCalled: false,
+		},
+		{
+			name:                  "retryable error: closed network connection (net package)",
+			inputErr:              net.ErrClosed,
+			expectedErrorIsCalled: false,
+		},
+		{
+			name:                  "some error",
+			inputErr:              errors.New("some error"),
+			expectedErrorIsCalled: true,
+			expectedErrorMsg:      "kgo.OnGroupManageError",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			log := &mockLogger{}
+			he := &hookEvent{log: log}
+			he.OnGroupManageError(tt.inputErr)
+			require.Equal(t, tt.expectedErrorIsCalled, log.errorIsCalled)
+			require.Equal(t, tt.expectedErrorMsg, log.errorMsg)
+		})
+	}
+}
+
+func TestHookEvent_OnBrokerConnect(t *testing.T) {
+	tests := []struct {
+		name                  string
+		inputErr              error
+		expectedErrorIsCalled bool
+		expectedErrorMsg      string
+	}{
+		{
+			name:                  "error is nil",
+			inputErr:              nil,
+			expectedErrorIsCalled: false,
+		},
+		{
+			name:                  "context canceled",
+			inputErr:              context.Canceled,
+			expectedErrorIsCalled: false,
+		},
+		{
+			name:                  "context deadline exceeded",
+			inputErr:              context.DeadlineExceeded,
+			expectedErrorIsCalled: false,
+		},
+		{
+			name:                  "retryable error: deadline exceeded (os package)",
+			inputErr:              os.ErrDeadlineExceeded,
+			expectedErrorIsCalled: false,
+		},
+		{
+			name:                  "retryable error: EOF (io package)",
+			inputErr:              io.EOF,
+			expectedErrorIsCalled: false,
+		},
+		{
+			name:                  "retryable error: closed network connection (net package)",
+			inputErr:              net.ErrClosed,
+			expectedErrorIsCalled: false,
+		},
+		{
+			name:                  "some error",
+			inputErr:              errors.New("some error"),
+			expectedErrorIsCalled: true,
+			expectedErrorMsg:      "kgo.OnBrokerConnect",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			log := &mockLogger{}
+			he := &hookEvent{log: log}
+			he.OnBrokerConnect(kgo.BrokerMetadata{}, 0, nil, tt.inputErr)
+			require.Equal(t, tt.expectedErrorIsCalled, log.errorIsCalled)
+			require.Equal(t, tt.expectedErrorMsg, log.errorMsg)
+		})
+	}
+}
+
+func TestHookEvent_OnBrokerWrite(t *testing.T) {
+	tests := []struct {
+		name                  string
+		inputErr              error
+		expectedErrorIsCalled bool
+		expectedErrorMsg      string
+	}{
+		{
+			name:                  "error is nil",
+			inputErr:              nil,
+			expectedErrorIsCalled: false,
+		},
+		{
+			name:                  "context canceled",
+			inputErr:              context.Canceled,
+			expectedErrorIsCalled: false,
+		},
+		{
+			name:                  "context deadline exceeded",
+			inputErr:              context.DeadlineExceeded,
+			expectedErrorIsCalled: false,
+		},
+		{
+			name:                  "retryable error: deadline exceeded (os package)",
+			inputErr:              os.ErrDeadlineExceeded,
+			expectedErrorIsCalled: false,
+		},
+		{
+			name:                  "retryable error: EOF (io package)",
+			inputErr:              io.EOF,
+			expectedErrorIsCalled: false,
+		},
+		{
+			name:                  "retryable error: closed network connection (net package)",
+			inputErr:              net.ErrClosed,
+			expectedErrorIsCalled: false,
+		},
+		{
+			name:                  "some error",
+			inputErr:              errors.New("some error"),
+			expectedErrorIsCalled: true,
+			expectedErrorMsg:      "kgo.OnBrokerWrite",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			log := &mockLogger{}
+			he := &hookEvent{log: log}
+			he.OnBrokerWrite(kgo.BrokerMetadata{}, 0, 0, 0, 0, tt.inputErr)
+			require.Equal(t, tt.expectedErrorIsCalled, log.errorIsCalled)
+			require.Equal(t, tt.expectedErrorMsg, log.errorMsg)
+		})
+	}
+}
+
+func TestHookEvent_OnBrokerRead(t *testing.T) {
+	tests := []struct {
+		name                  string
+		inputErr              error
+		expectedErrorIsCalled bool
+		expectedErrorMsg      string
+	}{
+		{
+			name:                  "error is nil",
+			inputErr:              nil,
+			expectedErrorIsCalled: false,
+		},
+		{
+			name:                  "context canceled",
+			inputErr:              context.Canceled,
+			expectedErrorIsCalled: false,
+		},
+		{
+			name:                  "context deadline exceeded",
+			inputErr:              context.DeadlineExceeded,
+			expectedErrorIsCalled: false,
+		},
+		{
+			name:                  "retryable error: deadline exceeded (os package)",
+			inputErr:              os.ErrDeadlineExceeded,
+			expectedErrorIsCalled: false,
+		},
+		{
+			name:                  "retryable error: EOF (io package)",
+			inputErr:              io.EOF,
+			expectedErrorIsCalled: false,
+		},
+		{
+			name:                  "retryable error: closed network connection (net package)",
+			inputErr:              net.ErrClosed,
+			expectedErrorIsCalled: false,
+		},
+		{
+			name:                  "some error",
+			inputErr:              errors.New("some error"),
+			expectedErrorIsCalled: true,
+			expectedErrorMsg:      "kgo.OnBrokerRead",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			log := &mockLogger{}
+			he := &hookEvent{log: log}
+			he.OnBrokerRead(kgo.BrokerMetadata{}, 0, 0, 0, 0, tt.inputErr)
+			require.Equal(t, tt.expectedErrorIsCalled, log.errorIsCalled)
+			require.Equal(t, tt.expectedErrorMsg, log.errorMsg)
+		})
+	}
+}
+
+func TestHookEvent_OnProduceRecordUnbuffered(t *testing.T) {
+	tests := []struct {
+		name                  string
+		inputErr              error
+		expectedErrorIsCalled bool
+		expectedErrorMsg      string
+	}{
+		{
+			name:                  "error is nil",
+			inputErr:              nil,
+			expectedErrorIsCalled: false,
+		},
+		{
+			name:                  "context canceled",
+			inputErr:              context.Canceled,
+			expectedErrorIsCalled: false,
+		},
+		{
+			name:                  "context deadline exceeded",
+			inputErr:              context.DeadlineExceeded,
+			expectedErrorIsCalled: false,
+		},
+		{
+			name:                  "retryable error: deadline exceeded (os package)",
+			inputErr:              os.ErrDeadlineExceeded,
+			expectedErrorIsCalled: false,
+		},
+		{
+			name:                  "retryable error: EOF (io package)",
+			inputErr:              io.EOF,
+			expectedErrorIsCalled: false,
+		},
+		{
+			name:                  "retryable error: closed network connection (net package)",
+			inputErr:              net.ErrClosed,
+			expectedErrorIsCalled: false,
+		},
+		{
+			name:                  "some error",
+			inputErr:              errors.New("some error"),
+			expectedErrorIsCalled: true,
+			expectedErrorMsg:      "kgo.OnProduceRecordUnbuffered",
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			log := &mockLogger{}
+			he := &hookEvent{log: log}
+			he.OnProduceRecordUnbuffered(&kgo.Record{}, tt.inputErr)
+			require.Equal(t, tt.expectedErrorIsCalled, log.errorIsCalled)
+			require.Equal(t, tt.expectedErrorMsg, log.errorMsg)
+		})
+	}
+}
+
+// Mocks
+
+type mockLogger struct {
+	errorIsCalled bool
+	errorMsg      string
+}
+
+func (m *mockLogger) Init(...logger.Option) error {
+	panic("implement me")
+}
+
+func (m *mockLogger) Clone(...logger.Option) logger.Logger {
+	panic("implement me")
+}
+
+func (m *mockLogger) V(logger.Level) bool {
+	panic("implement me")
+}
+
+func (m *mockLogger) Level(logger.Level) {
+	panic("implement me")
+}
+
+func (m *mockLogger) Options() logger.Options {
+	panic("implement me")
+}
+
+func (m *mockLogger) Fields(...interface{}) logger.Logger {
+	panic("implement me")
+}
+
+func (m *mockLogger) Info(context.Context, string, ...interface{}) {
+	panic("implement me")
+}
+
+func (m *mockLogger) Trace(context.Context, string, ...interface{}) {
+	panic("implement me")
+}
+
+func (m *mockLogger) Debug(context.Context, string, ...interface{}) {
+	panic("implement me")
+}
+
+func (m *mockLogger) Warn(context.Context, string, ...interface{}) {
+	panic("implement me")
+}
+
+func (m *mockLogger) Error(ctx context.Context, msg string, args ...interface{}) {
+	m.errorIsCalled = true
+	m.errorMsg = msg
+}
+
+func (m *mockLogger) Fatal(context.Context, string, ...interface{}) {
+	panic("implement me")
+}
+
+func (m *mockLogger) Log(context.Context, logger.Level, string, ...interface{}) {
+	panic("implement me")
+}
+
+func (m *mockLogger) Name() string {
+	panic("implement me")
+}
+
+func (m *mockLogger) String() string {
+	panic("implement me")
+}

kgo.go

@@ -19,6 +19,7 @@ import (
 	"go.unistack.org/micro/v3/metadata"
 	"go.unistack.org/micro/v3/semconv"
 	"go.unistack.org/micro/v3/tracer"
+	mjitter "go.unistack.org/micro/v3/util/jitter"
 	mrand "go.unistack.org/micro/v3/util/rand"
 )
 
@@ -59,12 +60,14 @@ type Broker struct {
 	c         *kgo.Client
 	connected *atomic.Uint32
 
+	done chan struct{}
+
 	kopts []kgo.Opt
 	subs  []*Subscriber
 
 	opts broker.Options
-
 	sync.RWMutex
+
 	init bool
 }
 
@@ -143,6 +146,25 @@ func (k *Broker) connect(ctx context.Context, opts ...kgo.Opt) (*kgo.Client, *ho
 			return nil, nil, err
 		}
 		k.connected.Store(1)
+
+		if fatalOnError {
+			go func() {
+				c := 3
+				n := 0
+				tc := mjitter.NewTicker(500*time.Millisecond, 1*time.Second)
+				defer tc.Stop()
+				for range tc.C {
+					if k.connected.Load() == 0 {
+						if n > c {
+							k.opts.Logger.Fatal(context.Background(), "broker fatal error")
+						}
+						n++
+					} else {
+						n = 0
+					}
+				}
+			}()
+		}
 		return c, htracer, nil
 	}
 }
@@ -204,6 +226,7 @@ func (k *Broker) Disconnect(ctx context.Context) error {
 	}
 
 	k.connected.Store(0)
+	close(k.done)
 	return nil
 }
 
@@ -257,18 +280,6 @@ func (k *Broker) Publish(ctx context.Context, topic string, msg *broker.Message,
 }
 
 func (k *Broker) publish(ctx context.Context, msgs []*broker.Message, opts ...broker.PublishOption) error {
-	k.Lock()
-	if k.connected.Load() == 0 {
-		c, _, err := k.connect(ctx, k.kopts...)
-		if err != nil {
-			k.Unlock()
-			return err
-		}
-		k.c = c
-		k.connected.Store(1)
-	}
-	k.Unlock()
-
 	options := broker.NewPublishOptions(opts...)
 	records := make([]*kgo.Record, 0, len(msgs))
 	var errs []string
@@ -391,12 +402,6 @@ func (k *Broker) Subscribe(ctx context.Context, topic string, handler broker.Han
 		}
 	}
 
-	if options.Context != nil {
-		if v, ok := options.Context.Value(fatalOnErrorKey{}).(bool); ok && v {
-			fatalOnError = v
-		}
-	}
-
 	sub := &Subscriber{
 		topic:        topic,
 		opts:         options,
@@ -492,5 +497,6 @@ func NewBroker(opts ...broker.Option) *Broker {
 		connected: &atomic.Uint32{},
 		opts:      options,
 		kopts:     kopts,
+		done:      make(chan struct{}),
 	}
 }

kgo_test.go

@@ -117,6 +117,10 @@ func TestFail(t *testing.T) {
 }
 
 func TestConnect(t *testing.T) {
+	if tr := os.Getenv("INTEGRATION_TESTS"); len(tr) > 0 {
+		t.Skip()
+	}
+
 	var addrs []string
 	ctx := context.TODO()
 	b := kgo.NewBroker(

subscriber.go

@@ -25,33 +25,43 @@ type tp struct {
 
 type consumer struct {
 	topic string
+
 	c         *kgo.Client
 	htracer   *hookTracer
+	connected *atomic.Uint32
+
 	quit chan struct{}
 	done chan struct{}
 	recs chan kgo.FetchTopicPartition
-	kopts     broker.Options
+
-	partition int32
-	opts      broker.SubscribeOptions
 	handler broker.Handler
-	connected *atomic.Uint32
+
+	kopts broker.Options
+	opts  broker.SubscribeOptions
+
+	partition int32
 }
 
 type Subscriber struct {
-	consumers map[tp]*consumer
-	c         *kgo.Client
-	htracer   *hookTracer
 	topic string
 
+	consumers map[tp]*consumer
+
+	c         *kgo.Client
+	htracer   *hookTracer
+	connected *atomic.Uint32
+
 	handler broker.Handler
+
 	done chan struct{}
+
 	kopts broker.Options
 	opts  broker.SubscribeOptions
 
-	connected *atomic.Uint32
-	sync.RWMutex
 	closed       bool
 	fatalOnError bool
+
+	sync.RWMutex
 }
 
 func (s *Subscriber) Client() *kgo.Client {
@@ -119,7 +129,7 @@ func (s *Subscriber) poll(ctx context.Context) {
 
 				s.Lock()
 				for p, l := range lmap {
-					s.kopts.Meter.Counter(semconv.BrokerGroupLag, "topic", s.topic, "group", s.opts.Group, "partition", strconv.Itoa(int(p)), "lag", strconv.Itoa(int(l.Lag)))
+					s.kopts.Meter.Counter(semconv.BrokerGroupLag, "topic", s.topic, "group", s.opts.Group, "partition", strconv.Itoa(int(p))).Set(uint64(l.Lag))
 				}
 				s.Unlock()
 
@@ -160,7 +170,10 @@ func (s *Subscriber) killConsumers(ctx context.Context, lost map[string][]int32)
 	for topic, partitions := range lost {
 		for _, partition := range partitions {
 			tps := tp{topic, partition}
-			pc := s.consumers[tps]
+			pc, ok := s.consumers[tps]
+			if !ok {
+				continue
+			}
 			delete(s.consumers, tps)
 			close(pc.quit)
 			if s.kopts.Logger.V(logger.DebugLevel) {
@@ -174,7 +187,7 @@ func (s *Subscriber) killConsumers(ctx context.Context, lost map[string][]int32)
 
 func (s *Subscriber) autocommit(_ *kgo.Client, _ *kmsg.OffsetCommitRequest, _ *kmsg.OffsetCommitResponse, err error) {
 	if err != nil {
-		s.connected.Store(0)
+		//		s.connected.Store(0)
 		if s.fatalOnError {
 			s.kopts.Logger.Fatal(context.TODO(), "kgo.AutoCommitCallback error", err)
 		}
@@ -261,11 +274,17 @@ func (pc *consumer) consume() {
 				} else if pc.opts.BodyOnly {
 					p.msg.Body = record.Value
 				} else {
+					if sp != nil {
 						sp.AddEvent("codec unmarshal start")
+					}
 					err := pc.kopts.Codec.Unmarshal(record.Value, p.msg)
+					if sp != nil {
 						sp.AddEvent("codec unmarshal stop")
+					}
 					if err != nil {
+						if sp != nil {
 							sp.SetStatus(tracer.SpanStatusError, err.Error())
+						}
 						pc.kopts.Meter.Counter(semconv.SubscribeMessageTotal, "endpoint", record.Topic, "topic", record.Topic, "status", "failure").Inc()
 						p.err = err
 						p.msg.Body = record.Value
@@ -276,7 +295,7 @@ func (pc *consumer) consume() {
 								pc.c.MarkCommitRecords(record)
 							} else {
 								eventPool.Put(p)
-								pc.connected.Store(0)
+								//								pc.connected.Store(0)
 								pc.kopts.Logger.Fatal(pc.kopts.Context, "[kgo] ErrLostMessage wtf?")
 								return
 							}
@@ -293,19 +312,27 @@ func (pc *consumer) consume() {
 						pc.kopts.Meter.Summary(semconv.SubscribeMessageLatencyMicroseconds, "endpoint", record.Topic, "topic", record.Topic).Update(te.Seconds())
 						pc.kopts.Meter.Histogram(semconv.SubscribeMessageDurationSeconds, "endpoint", record.Topic, "topic", record.Topic).Update(te.Seconds())
 						eventPool.Put(p)
-						pc.connected.Store(0)
+						//						pc.connected.Store(0)
 						pc.kopts.Logger.Fatal(pc.kopts.Context, "[kgo] Unmarshal err not handled wtf?")
+						if sp != nil {
 							sp.Finish()
+						}
 						return
 					}
 				}
+				if sp != nil {
 					sp.AddEvent("handler start")
+				}
 				err := pc.handler(p)
+				if sp != nil {
 					sp.AddEvent("handler stop")
+				}
 				if err == nil {
 					pc.kopts.Meter.Counter(semconv.SubscribeMessageTotal, "endpoint", record.Topic, "topic", record.Topic, "status", "success").Inc()
 				} else {
+					if sp != nil {
 						sp.SetStatus(tracer.SpanStatusError, err.Error())
+					}
 					pc.kopts.Meter.Counter(semconv.SubscribeMessageTotal, "endpoint", record.Topic, "topic", record.Topic, "status", "failure").Inc()
 				}
 				pc.kopts.Meter.Counter(semconv.SubscribeMessageInflight, "endpoint", record.Topic, "topic", record.Topic).Dec()
@@ -314,9 +341,13 @@ func (pc *consumer) consume() {
 				} else if err != nil {
 					p.err = err
 					if eh != nil {
+						if sp != nil {
 							sp.AddEvent("error handler start")
+						}
 						_ = eh(p)
+						if sp != nil {
 							sp.AddEvent("error handler stop")
+						}
 					} else {
 						if pc.kopts.Logger.V(logger.ErrorLevel) {
 							pc.kopts.Logger.Error(pc.kopts.Context, "[kgo]: subscriber error", err)
@@ -331,14 +362,18 @@ func (pc *consumer) consume() {
 					pc.c.MarkCommitRecords(record)
 				} else {
 					eventPool.Put(p)
-					pc.connected.Store(0)
+					//					pc.connected.Store(0)
 					pc.kopts.Logger.Fatal(pc.kopts.Context, "[kgo] ErrLostMessage wtf?")
+					if sp != nil {
 						sp.SetStatus(tracer.SpanStatusError, "ErrLostMessage")
 						sp.Finish()
+					}
 					return
 				}
+				if sp != nil {
 					sp.Finish()
 				}
 			}
 		}
+	}
 }

tracer.go

@@ -32,6 +32,9 @@ var (
 // the record's context, so it can be ended in the OnProduceRecordUnbuffered
 // hook.
 func (m *hookTracer) OnProduceRecordBuffered(r *kgo.Record) {
+	if !m.tracer.Enabled() {
+		return
+	}
 	// Set up span options.
 	attrs := []interface{}{
 		messagingSystem,
@@ -77,6 +80,9 @@ func (m *hookTracer) OnProduceRecordBuffered(r *kgo.Record) {
 // It sets attributes with values unset when producing and records any error
 // that occurred during the publish operation.
 func (m *hookTracer) OnProduceRecordUnbuffered(r *kgo.Record, err error) {
+	if !m.tracer.Enabled() {
+		return
+	}
 	if span, ok := tracer.SpanFromContext(r.Context); ok {
 		span.AddLabels(
 			semconv.MessagingKafkaDestinationPartition(int(r.Partition)),
@@ -96,6 +102,9 @@ func (m *hookTracer) OnProduceRecordUnbuffered(r *kgo.Record, err error) {
 // OnFetchRecordUnbuffered hook and can be used in downstream consumer
 // processing.
 func (m *hookTracer) OnFetchRecordBuffered(r *kgo.Record) {
+	if !m.tracer.Enabled() {
+		return
+	}
 	// Set up the span options.
 	attrs := []interface{}{
 		messagingSystem,
@@ -141,6 +150,9 @@ func (m *hookTracer) OnFetchRecordBuffered(r *kgo.Record) {
 // OnFetchRecordUnbuffered continues and ends the "receive" span for an
 // unbuffered record.
 func (m *hookTracer) OnFetchRecordUnbuffered(r *kgo.Record, _ bool) {
+	if !m.tracer.Enabled() {
+		return
+	}
 	span, _ := tracer.SpanFromContext(r.Context)
 	span.Finish()
 }
@@ -155,6 +167,13 @@ func (m *hookTracer) OnFetchRecordUnbuffered(r *kgo.Record, _ bool) {
 // not a record which has been created for producing, so call this at the start of each
 // iteration of your processing for the record.
 func (m *hookTracer) WithProcessSpan(r *kgo.Record) (context.Context, tracer.Span) {
+	if r.Context == nil {
+		r.Context = context.Background()
+	}
+
+	if !m.tracer.Enabled() {
+		return r.Context, nil
+	}
 	// Set up the span options.
 	attrs := []interface{}{
 		messagingSystem,