Merge pull request #79 from unistack-org/kgo-context

propogate context to kgo record

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,24 @@
+---
+name: Bug report
+about: For reporting bugs in go-micro
+title: "[BUG]"
+labels: ''
+assignees: ''
+
+---
+
+**Describe the bug**
+
+1. What are you trying to do?
+2. What did you expect to happen?
+3. What happens instead?
+
+**How to reproduce the bug:**
+
+If possible, please include a minimal code snippet here.
+
+**Environment:**
+Go Version: please paste `go version` output here
+```
+please paste `go env` output here
+```

.github/ISSUE_TEMPLATE/feature-request---enhancement.md

@@ -0,0 +1,17 @@
+---
+name: Feature request / Enhancement
+about: If you have a need not served by go-micro
+title: "[FEATURE]"
+labels: ''
+assignees: ''
+
+---
+
+**Is your feature request related to a problem? Please describe.**
+A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+
+**Describe the solution you'd like**
+A clear and concise description of what you want to happen.
+
+**Additional context**
+Add any other context or screenshots about the feature request here.

.github/ISSUE_TEMPLATE/question.md

@@ -0,0 +1,14 @@
+---
+name: Question
+about: Ask a question about go-micro
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+Before asking, please check if your question has already been answered: 
+
+1. Check the documentation - https://micro.mu/docs/
+2. Check the examples and plugins - https://github.com/micro/examples & https://github.com/micro/go-plugins
+3. Search existing issues

.github/PULL_REQUEST_TEMPLATE.md

@@ -0,0 +1,9 @@
+## Pull Request template
+Please, go through these steps before clicking submit on this PR.
+
+1. Give a descriptive title to your PR.
+2. Provide a description of your changes.
+3. Make sure you have some relevant tests.
+4. Put `closes #XXXX` in your comment to auto-close the issue that your PR fixes (if applicable).
+
+**PLEASE REMOVE THIS TEMPLATE BEFORE SUBMITTING**

.github/dependabot.yml

@@ -0,0 +1,19 @@
+# To get started with Dependabot version updates, you'll need to specify which
+# package ecosystems to update and where the package manifests are located.
+# Please see the documentation for all configuration options:
+# https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
+
+version: 2
+updates:
+
+  # Maintain dependencies for GitHub Actions
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "daily"
+
+  # Maintain dependencies for Golang
+  - package-ecosystem: "gomod"
+    directory: "/"
+    schedule:
+      interval: "daily"

.github/workflows/autoapprove.yml

@@ -0,0 +1,20 @@
+name: "autoapprove"
+
+on:
+  pull_request_target:
+    types: [assigned, opened, synchronize, reopened]
+
+permissions:
+  pull-requests: write
+  contents: write
+
+jobs:
+  autoapprove:
+    runs-on: ubuntu-latest
+    steps:
+      - name: approve
+        uses: hmarr/auto-approve-action@v2
+        if: github.actor == 'vtolstov' || github.actor == 'dependabot[bot]'
+        id: approve
+        with:
+          github-token: ${{ secrets.GITHUB_TOKEN }}

.github/workflows/automerge.yml

@@ -0,0 +1,21 @@
+name: "automerge"
+
+on:
+  pull_request_target:
+    types: [assigned, opened, synchronize, reopened]
+
+permissions:
+  pull-requests: write
+  contents: write
+
+jobs:
+  automerge:
+    runs-on: ubuntu-latest
+    if: github.actor == 'vtolstov'
+    steps:
+      - name: merge
+        id: merge
+        run: gh pr merge --auto --merge "$PR_URL"
+        env:
+          PR_URL: ${{github.event.pull_request.html_url}}
+          GITHUB_TOKEN: ${{secrets.TOKEN}}

.github/workflows/build.yml

@@ -0,0 +1,47 @@
+name: build
+on:
+ push:
+    branches:
+    - master
+    - v3
+jobs:
+  test:
+    name: test
+    runs-on: ubuntu-latest
+    steps:
+    - name: setup
+      uses: actions/setup-go@v3
+      with:
+        go-version: 1.17
+    - name: checkout
+      uses: actions/checkout@v3
+    - name: cache
+      uses: actions/cache@v3
+      with:
+        path: ~/go/pkg/mod
+        key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
+        restore-keys: ${{ runner.os }}-go-
+    - name: deps
+      run: go get -v -t -d ./...
+    - name: test
+      env:
+        INTEGRATION_TESTS: yes
+      run: go test -mod readonly -v ./...
+  lint:
+    name: lint
+    runs-on: ubuntu-latest
+    steps:
+      - name: checkout
+        uses: actions/checkout@v3
+      - name: lint
+        uses: golangci/golangci-lint-action@v3.2.0
+        continue-on-error: true
+        with:
+          # Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version.
+          version: v1.30
+          # Optional: working directory, useful for monorepos
+          # working-directory: somedir
+          # Optional: golangci-lint command line arguments.
+          # args: --issues-exit-code=0
+          # Optional: show only new issues if it's a pull request. The default value is `false`.
+          # only-new-issues: true

.github/workflows/codeql-analysis.yml

@@ -0,0 +1,78 @@
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: "codeql"
+
+on:
+  workflow_run:
+    workflows: ["prbuild"]
+    types:
+      - completed
+  push:
+    branches: [ master, v3 ]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [ master, v3 ]
+  schedule:
+    - cron: '34 1 * * 0'
+
+jobs:
+  analyze:
+    name: analyze
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [ 'go' ]
+        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ]
+        # Learn more:
+        # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed
+
+    steps:
+    - name: checkout
+      uses: actions/checkout@v3
+    - name: setup
+      uses: actions/setup-go@v3
+      with:
+        go-version: 1.17
+    # Initializes the CodeQL tools for scanning.
+    - name: init
+      uses: github/codeql-action/init@v2
+      with:
+        languages: ${{ matrix.language }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+        # queries: ./path/to/local/query, your-org/your-repo/queries@main
+
+    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
+    # If this step fails, then you should remove it and run the build manually (see below)
+    - name: autobuild
+      uses: github/codeql-action/autobuild@v2
+
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 https://git.io/JvXDl
+
+    # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
+    #    and modify them (or add more) to build your code if your project
+    #    uses a compiled language
+
+    #- run: |
+    #   make bootstrap
+    #   make release
+
+    - name: analyze
+      uses: github/codeql-action/analyze@v2

.github/workflows/dependabot-automerge.yml

@@ -0,0 +1,27 @@
+name: "dependabot-automerge"
+
+on:
+  pull_request_target:
+    types: [assigned, opened, synchronize, reopened]
+
+permissions:
+  pull-requests: write
+  contents: write
+
+jobs:
+  automerge:
+    runs-on: ubuntu-latest
+    if: github.actor == 'dependabot[bot]'
+    steps:
+      - name: metadata
+        id: metadata
+        uses: dependabot/fetch-metadata@v1.3.4
+        with:
+          github-token: "${{ secrets.TOKEN }}"
+      - name: merge
+        id: merge
+        if: ${{contains(steps.metadata.outputs.dependency-names, 'go.unistack.org')}}
+        run: gh pr merge --auto --merge "$PR_URL"
+        env:
+          PR_URL: ${{github.event.pull_request.html_url}}
+          GITHUB_TOKEN: ${{secrets.TOKEN}}

.github/workflows/pr.yml

@@ -0,0 +1,47 @@
+name: prbuild
+on:
+  pull_request:
+    branches:
+    - master
+    - v3
+jobs:
+  test:
+    name: test
+    runs-on: ubuntu-latest
+    steps:
+    - name: setup
+      uses: actions/setup-go@v3
+      with:
+        go-version: 1.17
+    - name: checkout
+      uses: actions/checkout@v3
+    - name: cache
+      uses: actions/cache@v3
+      with:
+        path: ~/go/pkg/mod
+        key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
+        restore-keys: ${{ runner.os }}-go-
+    - name: deps
+      run: go get -v -t -d ./...
+    - name: test
+      env:
+        INTEGRATION_TESTS: yes
+      run: go test -mod readonly -v ./...
+  lint:
+    name: lint
+    runs-on: ubuntu-latest
+    steps:
+      - name: checkout
+        uses: actions/checkout@v3
+      - name: lint
+        uses: golangci/golangci-lint-action@v3.2.0
+        continue-on-error: true
+        with:
+          # Required: the version of golangci-lint is required and must be specified without patch version: we always use the latest patch version.
+          version: v1.30
+          # Optional: working directory, useful for monorepos
+          # working-directory: somedir
+          # Optional: golangci-lint command line arguments.
+          # args: --issues-exit-code=0
+          # Optional: show only new issues if it's a pull request. The default value is `false`.
+          # only-new-issues: true

README.md

@@ -1,2 +1,9 @@
 # micro-broker-kgo
 yet another micro kafka broker alternative
+
+TODO:
+* dont always append options from context on Init and New
+* add SubscriberOptions(...kgo.Opt)
+* add ServerSubscribeOptions(...kgo.Opt)
+* check PublisherOptions(...kgo.Opt)
+* check ClientPublisherOptions(...kgo.Opt)

go.mod

@@ -1,13 +1,19 @@
-module github.com/unistack-org/micro-broker-kgo/v3
+module go.unistack.org/micro-broker-kgo/v3
 
-go 1.16
+go 1.17
 
 require (
-	github.com/twmb/franz-go v0.10.3-0.20210825060253-d5e80b38ca2b
+	github.com/twmb/franz-go v1.8.0
-	github.com/twmb/franz-go/pkg/kmsg v0.0.0-20210826221812-c6df11da978a
+	github.com/twmb/franz-go/pkg/kmsg v1.2.0
-	github.com/unistack-org/micro-codec-json/v3 v3.2.5
+	go.unistack.org/micro/v3 v3.9.11
-	github.com/unistack-org/micro-proto v0.0.8 // indirect
+)
-	github.com/unistack-org/micro/v3 v3.6.3
+
-	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
+require (
-	google.golang.org/protobuf v1.27.1 // indirect
+	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/google/gnostic v0.6.8 // indirect
+	github.com/klauspost/compress v1.15.9 // indirect
+	github.com/pierrec/lz4/v4 v4.1.15 // indirect
+	go.unistack.org/micro-proto/v3 v3.2.7 // indirect
+	google.golang.org/protobuf v1.28.0 // indirect
+	gopkg.in/yaml.v3 v3.0.0 // indirect
 )

go.sum

@@ -1,123 +1,149 @@
 cloud.google.com/go v0.26.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
+cloud.google.com/go v0.34.0/go.mod h1:aQUYkXzVsufM+DwF1aE+0xfcU+56JwCaLick0ClmMTw=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
+github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU=
+github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY=
+github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0=
 github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU=
+github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc=
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
-github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
+github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc=
+github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk=
+github.com/cncf/xds/go v0.0.0-20210312221358-fbca930ec8ed/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs=
+github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E=
 github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
+github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
-github.com/ef-ds/deque v1.0.4/go.mod h1:gXDnTC3yqvBcHbq2lcExjtAcVrOnJCbMcZXmuj8Z4tg=
+github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE=
+github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
+github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
+github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.mod h1:cXg6YxExXjJnVBQHBLXeUAgxn2UodCpnH306RInaBQk=
+github.com/envoyproxy/go-control-plane v0.9.9-0.20210512163311-63b5d3c536b0/go.mod h1:hliV/p42l8fGbc6Y9bQ70uLwIvmJyVE5k4iMKlh8wCQ=
 github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c=
-github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg=
+github.com/flowstack/go-jsonschema v0.1.1/go.mod h1:yL7fNggx1o8rm9RlgXv7hTBWxdBM0rVwpMwimd3F3N0=
+github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
 github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A=
 github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
 github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U=
+github.com/golang/protobuf v1.3.3/go.mod h1:vzj43D7+SQXF/4pzW/hwtAqwc6iTitCiVSaWz5lYuqw=
 github.com/golang/protobuf v1.4.0-rc.1/go.mod h1:ceaxUfeHdC40wWswd/P6IGgMaK3YpKi5j83Wpe3EHw8=
 github.com/golang/protobuf v1.4.0-rc.1.0.20200221234624-67d41d38c208/go.mod h1:xKAWHe0F5eneWXFV3EuXVDTCmh+JuBKY0li0aMyXATA=
 github.com/golang/protobuf v1.4.0-rc.2/go.mod h1:LlEzMj4AhA7rCAGe4KMBDvJI+AwstrUpVNzEA03Pprs=
 github.com/golang/protobuf v1.4.0-rc.4.0.20200313231945-b860323f09d0/go.mod h1:WU3c8KckQ9AFe+yFwt9sWVRKCVIyN9cPHBJSNnbL67w=
 github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvqG2KuDX0=
 github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8=
+github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
 github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI=
 github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
-github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw=
-github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
+github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY=
-github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
+github.com/google/gnostic v0.6.6/go.mod h1:Nm8234We1lq6iB9OmlgNv3nH91XLLVZHCDayfA3xq+E=
+github.com/google/gnostic v0.6.8 h1:bT56GPYBWh1tvBuBEd94qcS3+60b+y0HQur0ITkGuCk=
+github.com/google/gnostic v0.6.8/go.mod h1:Nm8234We1lq6iB9OmlgNv3nH91XLLVZHCDayfA3xq+E=
 github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M=
 github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
 github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
 github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
+github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o=
-github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4=
+github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE=
-github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM=
+github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
-github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
+github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
-github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
+github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg=
-github.com/jcmturner/aescts/v2 v2.0.0/go.mod h1:AiaICIRyfYg35RUkr8yESTqvSy7csK90qZ5xfvvsoNs=
+github.com/klauspost/compress v1.15.9 h1:wKRjX6JRtDdrE9qwa4b/Cip7ACOshUI4smpCQanqjSY=
-github.com/jcmturner/dnsutils/v2 v2.0.0/go.mod h1:b0TnjGOvI/n42bZa+hmXL+kFJZsFT7G4t3HTlQ184QM=
+github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
-github.com/jcmturner/gofork v1.0.0/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/UM3ncEo0o=
+github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
-github.com/jcmturner/goidentity/v6 v6.0.1/go.mod h1:X1YW3bgtvwAXju7V3LCIMpY0Gbxyjn/mY9zx4tFonSg=
+github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI=
-github.com/jcmturner/gokrb5/v8 v8.4.2/go.mod h1:sb+Xq/fTY5yktf/VxLsE3wlfPqQjp0aWNYyvBVK62bc=
+github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
-github.com/jcmturner/rpc/v2 v2.0.3/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc=
+github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ=
-github.com/klauspost/compress v1.13.4 h1:0zhec2I8zGnjWcKyLl6i3gPqKANCCn5e9xmviEEeX6s=
+github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI=
-github.com/klauspost/compress v1.13.4/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg=
+github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY=
+github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE=
 github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
-github.com/pierrec/lz4/v4 v4.1.8 h1:ieHkV+i2BRzngO4Wd/3HGowuZStgq6QkPsD1eolNAO4=
+github.com/pierrec/lz4/v4 v4.1.15 h1:MO0/ucJhngq7299dKLwIMtgTfbkoSPF6AoMYDd8Q4q0=
-github.com/pierrec/lz4/v4 v4.1.8/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
+github.com/pierrec/lz4/v4 v4.1.15/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
-github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
-github.com/silas/dag v0.0.0-20210121180416-41cf55125c34/go.mod h1:7RTUFBdIRC9nZ7/3RyRNH1bdqIShrDejd1YbLwgPS+I=
+github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ=
+github.com/silas/dag v0.0.0-20211117232152-9d50aa809f35/go.mod h1:7RTUFBdIRC9nZ7/3RyRNH1bdqIShrDejd1YbLwgPS+I=
+github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA=
+github.com/stoewer/go-strcase v1.2.0/go.mod h1:IBiWB2sKIp3wVVQ3Y035++gc+knqhUQag1KpM8ahLw8=
 github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
-github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
+github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
-github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
-github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
+github.com/twmb/franz-go v1.8.0 h1:58lABTO3YK179YmF+z/3EqL17ZbqhrmEDEdVHcd5tdA=
-github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
+github.com/twmb/franz-go v1.8.0/go.mod h1:PMze0jNfNghhih2XHbkmTFykbMF5sJqmNJB31DOOzro=
-github.com/twmb/franz-go v0.10.3-0.20210825060253-d5e80b38ca2b h1:t7+Qdd/GHtUBN6D71CwwOEfsZmK0QQ/up3M6gaoaHQ4=
+github.com/twmb/franz-go/pkg/kmsg v1.2.0 h1:jYWh2qFw5lDbNv5Gvu/sMKagzICxuA5L6m1W2Oe7XUo=
-github.com/twmb/franz-go v0.10.3-0.20210825060253-d5e80b38ca2b/go.mod h1:Txc5/v0DIKGcdCa1VZhyasECMJ4svN/LHUKYUPYihL0=
+github.com/twmb/franz-go/pkg/kmsg v1.2.0/go.mod h1:SxG/xJKhgPu25SamAq0rrucfp7lbzCpEXOC+vH/ELrY=
-github.com/twmb/franz-go/pkg/kmsg v0.0.0-20210823212011-0d01f7456b4d/go.mod h1:SxG/xJKhgPu25SamAq0rrucfp7lbzCpEXOC+vH/ELrY=
+github.com/xeipuuv/gojsonpointer v0.0.0-20180127040702-4e3ac2762d5f/go.mod h1:N2zxlSyiKSe5eX1tZViRH5QA0qijqEDrYZiPEAiq3wU=
-github.com/twmb/franz-go/pkg/kmsg v0.0.0-20210826221812-c6df11da978a h1:UBO58MRI/GH5eD5+fKXZoJMGyEcsUs9LQnHJbNL0yKs=
+github.com/xeipuuv/gojsonreference v0.0.0-20180127040603-bd5ef7bd5415/go.mod h1:GwrjFmJcFw6At/Gs6z4yjiIwzuJ1/+UwLxMQDVQXShQ=
-github.com/twmb/franz-go/pkg/kmsg v0.0.0-20210826221812-c6df11da978a/go.mod h1:SxG/xJKhgPu25SamAq0rrucfp7lbzCpEXOC+vH/ELrY=
+github.com/xeipuuv/gojsonschema v1.2.0/go.mod h1:anYRn/JVcOK2ZgGU+IjEV4nwlhoK5sQluxsYJ78Id3Y=
-github.com/twmb/go-rbtree v1.0.0 h1:KxN7dXJ8XaZ4cvmHV1qqXTshxX3EBvX/toG5+UR49Mg=
+go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
-github.com/twmb/go-rbtree v1.0.0/go.mod h1:UlIAI8gu3KRPkXSobZnmJfVwCJgEhD/liWzT5ppzIyc=
+go.unistack.org/micro-proto/v3 v3.2.7 h1:zG6d69kHc+oij2lwQ3AfrCgdjiEVRG2A7TlsxjusWs4=
-github.com/unistack-org/micro-codec-json/v3 v3.2.5 h1:WOilhbL0YSu58iIQIIxpawRYZyx6CR16tCpbX4ai3Vc=
+go.unistack.org/micro-proto/v3 v3.2.7/go.mod h1:ZltVWNECD5yK+40+OCONzGw4OtmSdTpVi8/KFgo9dqM=
-github.com/unistack-org/micro-codec-json/v3 v3.2.5/go.mod h1:LSzfrD9GYWCl6KOyihywx1wlbOgStrpyy3NVHNZAvHA=
+go.unistack.org/micro/v3 v3.9.11 h1:dsZVss3nvfByL1ZDJNnUVQB1N8w6qn4pr9vIkeBiii8=
-github.com/unistack-org/micro-proto v0.0.5/go.mod h1:EuI7UlfGXmT1hy6WacULib9LbNgRnDYQvTCFoLgKM2I=
+go.unistack.org/micro/v3 v3.9.11/go.mod h1:LrBm9Fsf4MbY8DlUbdwRTYJosMkN8wrtOQHoYBlMkz8=
-github.com/unistack-org/micro-proto v0.0.8 h1:g4UZGQGeYGI3CFJtjuEm47aouYPviG8SDhSifl0831w=
-github.com/unistack-org/micro-proto v0.0.8/go.mod h1:GYO53DWmeldRIo90cAdQx8bLr/WJMxW62W4ja74p1Ac=
-github.com/unistack-org/micro/v3 v3.3.19/go.mod h1:LXmPfbJnJNvL0kQs8HfnkV3Wya2Wb+C7keVq++RCZnk=
-github.com/unistack-org/micro/v3 v3.6.3 h1:CvC4B2kOgwzhPx+w9fcbEIJkgzxS7i84PRB+vL6Vz0U=
-github.com/unistack-org/micro/v3 v3.6.3/go.mod h1:Hp+DmX1Bt+/z+ihk5coYwNtXhPMwr4SJuCFRy2kyUl8=
 golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
-golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
+golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
-golang.org/x/crypto v0.0.0-20210817164053-32db794688a5/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
+golang.org/x/crypto v0.0.0-20220817201139-bc19a97f63c8/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=
 golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU=
 golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc=
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
+golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190213061140-3a22650c66bd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
 golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg=
-golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
+golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
-golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
+golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM=
-golang.org/x/net v0.0.0-20210510120150-4163338589ed/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20210805182204-aaa1db679c0d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
-golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
+golang.org/x/net v0.0.0-20211112202133-69e39bad7dc2/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
+golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
-golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
-golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
 golang.org/x/sys v0.0.0-20210423082822-04245dca01da/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
+golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
 golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
+golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
 golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
 golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY=
 golang.org/x/tools v0.0.0-20190311212946-11955173bddd/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs=
 golang.org/x/tools v0.0.0-20190524140312-2c0ae7006135/go.mod h1:RgjU9mgBXZiqYHBnxXauZ1Gv1EHHAz9KjViQ78xBX0Q=
-golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
 golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE=
+golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
 google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
 google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc=
 google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
+google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c=
 google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
+google.golang.org/genproto v0.0.0-20220107163113-42d7afdf6368/go.mod h1:5CzLGKJ67TSI2B9POpiiyGha0AjJvZIUgRMt1dSmuhc=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=
 google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=
+google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQciAY=
 google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk=
+google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0=
+google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU=
+google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34=
 google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8=
 google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0=
 google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
@@ -129,12 +155,17 @@ google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpAD
 google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c=
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
-google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ=
 google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
+google.golang.org/protobuf v1.28.0 h1:w43yiav+6bVFTBQFZX0r7ipe9JQ1QsbMgHwbBziscLw=
+google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15 h1:YR8cESwS4TdDjEe65xsg0ogRM/Nc3DYOhEAlW+xobZo=
+gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
+gopkg.in/yaml.v2 v2.2.3/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
+gopkg.in/yaml.v3 v3.0.0-20200615113413-eeeca48fe776/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
-gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
+gopkg.in/yaml.v3 v3.0.0 h1:hjy8E9ON/egN1tAYqKb61G10WtihqetD4sz2H+8nIeA=
+gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
 honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=
 honnef.co/go/tools v0.0.0-20190523083050-ea95bdfd59fc/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4=

kgo.go

@@ -1,31 +1,25 @@
 // Package kgo provides a kafka broker using kgo
-package kgo
+package kgo // import "go.unistack.org/micro-broker-kgo/v3"
 
 import (
 	"context"
 	"fmt"
-	"math/rand"
 	"strings"
 	"sync"
-	"sync/atomic"
 	"time"
 
-	kerr "github.com/twmb/franz-go/pkg/kerr"
+	"github.com/twmb/franz-go/pkg/kerr"
 	kgo "github.com/twmb/franz-go/pkg/kgo"
-	kmsg "github.com/twmb/franz-go/pkg/kmsg"
+	"github.com/twmb/franz-go/pkg/kmsg"
-	"github.com/unistack-org/micro/v3/broker"
+	"github.com/twmb/franz-go/pkg/kversion"
-	"github.com/unistack-org/micro/v3/logger"
+	"go.unistack.org/micro/v3/broker"
-	"github.com/unistack-org/micro/v3/metadata"
+	"go.unistack.org/micro/v3/logger"
-	"github.com/unistack-org/micro/v3/util/id"
+	"go.unistack.org/micro/v3/metadata"
-	mrand "github.com/unistack-org/micro/v3/util/rand"
+	"go.unistack.org/micro/v3/util/id"
-	"golang.org/x/sync/errgroup"
+	mrand "go.unistack.org/micro/v3/util/rand"
 )
 
-var pPool = sync.Pool{
+var _ broker.Broker = &kBroker{}
-	New: func() interface{} {
-		return &publication{msg: &broker.Message{}}
-	},
-}
 
 type kBroker struct {
 	writer    *kgo.Client // used only to push messages
@@ -46,6 +40,7 @@ type subscriber struct {
 	batchhandler broker.BatchHandler
 	closed       bool
 	done         chan struct{}
+	consumers    map[string]map[int32]worker
 	sync.RWMutex
 }
 
@@ -57,10 +52,6 @@ type publication struct {
 	ack bool
 }
 
-func init() {
-	rand.Seed(time.Now().UnixNano())
-}
-
 func (p *publication) Topic() string {
 	return p.topic
 }
@@ -128,7 +119,8 @@ func (k *kBroker) Connect(ctx context.Context) error {
 	kaddrs := k.opts.Addrs
 
 	// shuffle addrs
-	rand.Shuffle(len(kaddrs), func(i, j int) {
+	var rng mrand.Rand
+	rng.Shuffle(len(kaddrs), func(i, j int) {
 		kaddrs[i], kaddrs[j] = kaddrs[j], kaddrs[i]
 	})
 
@@ -142,6 +134,23 @@ func (k *kBroker) Connect(ctx context.Context) error {
 		if err != nil {
 			return err
 		}
+
+		// Request versions in order to guess Kafka Cluster version
+		versionsReq := kmsg.NewApiVersionsRequest()
+		versionsRes, err := versionsReq.RequestWith(ctx, c)
+		if err != nil {
+			return fmt.Errorf("failed to request api versions: %w", err)
+		}
+		err = kerr.ErrorForCode(versionsRes.ErrorCode)
+		if err != nil {
+			return fmt.Errorf("failed to request api versions. Inner kafka error: %w", err)
+		}
+		versions := kversion.FromApiVersionsResponse(versionsRes)
+
+		if k.opts.Logger.V(logger.InfoLevel) {
+			logger.Infof(ctx, "[kgo] connected to to kafka cluster version %v", versions.VersionGuess())
+		}
+
 		k.Lock()
 		k.connected = true
 		k.writer = c
@@ -234,23 +243,40 @@ func (k *kBroker) Publish(ctx context.Context, topic string, msg *broker.Message
 }
 
 func (k *kBroker) publish(ctx context.Context, msgs []*broker.Message, opts ...broker.PublishOption) error {
+	k.RLock()
+	if !k.connected {
+		k.RUnlock()
+		return broker.ErrNotConnected
+	}
+	k.RUnlock()
 	options := broker.NewPublishOptions(opts...)
 	records := make([]*kgo.Record, 0, len(msgs))
 	var errs []string
 	var err error
-	var buf []byte
+	var key []byte
+
+	if options.Context != nil {
+		if k, ok := options.Context.Value(publishKey{}).([]byte); ok && k != nil {
+			key = k
+		}
+	}
 
 	for _, msg := range msgs {
-		if options.BodyOnly {
+		rec := &kgo.Record{Context: ctx, Key: key}
-			buf = msg.Body
+		rec.Topic, _ = msg.Header.Get(metadata.HeaderTopic)
+		if k.opts.Codec.String() == "noop" {
+			rec.Value = msg.Body
+			for k, v := range msg.Header {
+				rec.Headers = append(rec.Headers, kgo.RecordHeader{Key: k, Value: []byte(v)})
+			}
+		} else if options.BodyOnly {
+			rec.Value = msg.Body
 		} else {
-			buf, err = k.opts.Codec.Marshal(msg)
+			rec.Value, err = k.opts.Codec.Marshal(msg)
 			if err != nil {
 				return err
 			}
 		}
-		topic, _ := msg.Header.Get(metadata.HeaderTopic)
-		rec := &kgo.Record{Value: buf, Topic: topic}
 		records = append(records, rec)
 	}
 
@@ -260,6 +286,7 @@ func (k *kBroker) publish(ctx context.Context, msgs []*broker.Message, opts ...b
 			errs = append(errs, result.Err.Error())
 		}
 	}
+
 	if len(errs) > 0 {
 		return fmt.Errorf("publish error: %s", strings.Join(errs, "\n"))
 	}
@@ -327,22 +354,42 @@ func (k *kBroker) Subscribe(ctx context.Context, topic string, handler broker.Ha
 	kaddrs := k.opts.Addrs
 
 	// shuffle addrs
-	rand.Shuffle(len(kaddrs), func(i, j int) {
+	var rng mrand.Rand
+	rng.Shuffle(len(kaddrs), func(i, j int) {
 		kaddrs[i], kaddrs[j] = kaddrs[j], kaddrs[i]
 	})
 
+	td := DefaultCommitInterval
+	if k.opts.Context != nil {
+		if v, ok := k.opts.Context.Value(commitIntervalKey{}).(time.Duration); ok && v > 0 {
+			td = v
+		}
+	}
+
+	sub := &subscriber{
+		topic:     topic,
+		done:      make(chan struct{}),
+		opts:      options,
+		handler:   handler,
+		kopts:     k.opts,
+		consumers: make(map[string]map[int32]worker),
+	}
+
 	kopts := append(k.kopts,
 		kgo.SeedBrokers(kaddrs...),
 		kgo.ConsumerGroup(options.Group),
 		kgo.ConsumeTopics(topic),
 		kgo.ConsumeResetOffset(kgo.NewOffset().AtStart()),
-		kgo.DisableAutoCommit(),
 		kgo.FetchMaxWait(1*time.Second),
 		// kgo.KeepControlRecords(),
 		kgo.Balancers(kgo.CooperativeStickyBalancer(), kgo.StickyBalancer()),
 		kgo.FetchIsolationLevel(kgo.ReadUncommitted()),
 		kgo.WithHooks(&metrics{meter: k.opts.Meter}),
-	// TODO: must set https://pkg.go.dev/github.com/twmb/franz-go/pkg/kgo#OnRevoked
+		kgo.AutoCommitMarks(),
+		kgo.AutoCommitInterval(td),
+		kgo.OnPartitionsAssigned(sub.assigned),
+		kgo.OnPartitionsRevoked(sub.revoked),
+		kgo.OnPartitionsLost(sub.revoked),
 	)
 
 	reader, err := kgo.NewClient(kopts...)
@@ -350,7 +397,7 @@ func (k *kBroker) Subscribe(ctx context.Context, topic string, handler broker.Ha
 		return nil, err
 	}
 
-	sub := &subscriber{topic: topic, done: make(chan struct{}), opts: options, reader: reader, handler: handler, kopts: k.opts}
+	sub.reader = reader
 	go sub.run(ctx)
 
 	k.Lock()
@@ -359,203 +406,18 @@ func (k *kBroker) Subscribe(ctx context.Context, topic string, handler broker.Ha
 	return sub, nil
 }
 
-func (s *subscriber) run(ctx context.Context) {
-	for {
-		select {
-		case <-ctx.Done():
-			return
-		case <-s.kopts.Context.Done():
-			return
-		default:
-			fetches := s.reader.PollFetches(ctx)
-			if fetches.IsClientClosed() {
-				// TODO: fatal ?
-				return
-			}
-			if len(fetches.Errors()) > 0 {
-				for _, err := range fetches.Errors() {
-					s.kopts.Logger.Errorf(ctx, "fetch err topic %s partition %d: %v", err.Topic, err.Partition, err.Err)
-				}
-				// TODO: fatal ?
-				return
-			}
-
-			if err := s.handleFetches(ctx, fetches); err != nil {
-				s.kopts.Logger.Errorf(ctx, "fetch handler err: %v", err)
-				// TODO: fatal ?
-				// return
-			}
-		}
-	}
-}
-
-func (s *subscriber) handleFetches(ctx context.Context, fetches kgo.Fetches) error {
-	var err error
-
-	eh := s.kopts.ErrorHandler
-	if s.opts.ErrorHandler != nil {
-		eh = s.opts.ErrorHandler
-	}
-
-	var mu sync.Mutex
-
-	done := int32(0)
-	doneCh := make(chan struct{})
-	g, gctx := errgroup.WithContext(ctx)
-
-	td := DefaultCommitInterval
-	if s.kopts.Context != nil {
-		if v, ok := s.kopts.Context.Value(commitIntervalKey{}).(time.Duration); ok && v > 0 {
-			td = v
-		}
-	}
-
-	// ticker for commit offsets
-	ticker := time.NewTicker(td)
-	defer ticker.Stop()
-
-	offsets := make(map[string]map[int32]kgo.EpochOffset)
-	offsets[s.topic] = make(map[int32]kgo.EpochOffset)
-
-	fillOffsets := func(off map[string]map[int32]kgo.EpochOffset, rec *kgo.Record) {
-		mu.Lock()
-		if at, ok := off[s.topic][rec.Partition]; ok {
-			if at.Epoch > rec.LeaderEpoch || at.Epoch == rec.LeaderEpoch && at.Offset > rec.Offset {
-				mu.Unlock()
-				return
-			}
-		}
-		off[s.topic][rec.Partition] = kgo.EpochOffset{Epoch: rec.LeaderEpoch, Offset: rec.Offset + 1}
-		mu.Unlock()
-	}
-
-	commitOffsets := func(cl *kgo.Client, ctx context.Context, off map[string]map[int32]kgo.EpochOffset) error {
-		var rerr error
-
-		mu.Lock()
-		offsets := off
-		mu.Unlock()
-
-		cl.CommitOffsetsSync(ctx, offsets, func(_ *kgo.Client, _ *kmsg.OffsetCommitRequest, resp *kmsg.OffsetCommitResponse, err error) {
-			if err != nil {
-				rerr = err
-				return
-			}
-
-			for _, topic := range resp.Topics {
-				for _, partition := range topic.Partitions {
-					if err := kerr.ErrorForCode(partition.ErrorCode); err != nil {
-						rerr = err
-						return
-					}
-				}
-			}
-		})
-
-		return rerr
-	}
-
-	go func() {
-		for {
-			select {
-			case <-gctx.Done():
-				return
-			case <-s.done:
-				atomic.StoreInt32(&done, 1)
-				if err := commitOffsets(s.reader, ctx, offsets); err != nil && s.kopts.Logger.V(logger.ErrorLevel) {
-					s.kopts.Logger.Errorf(s.kopts.Context, "[kgo]: failed to commit offsets: %v", err)
-				}
-				return
-			case <-doneCh:
-				return
-			case <-ticker.C:
-				if err := commitOffsets(s.reader, ctx, offsets); err != nil {
-					if s.kopts.Logger.V(logger.ErrorLevel) {
-						s.kopts.Logger.Errorf(s.kopts.Context, "[kgo]: failed to commit offsets: %v", err)
-					}
-					return
-				}
-			}
-		}
-	}()
-
-	for _, fetch := range fetches {
-		for _, ftopic := range fetch.Topics {
-			for _, partition := range ftopic.Partitions {
-				precords := partition.Records
-				g.Go(func() error {
-					for _, record := range precords {
-						if atomic.LoadInt32(&done) == 1 {
-							return nil
-						}
-						p := pPool.Get().(*publication)
-						p.msg.Header = nil
-						p.msg.Body = nil
-						p.topic = s.topic
-						p.err = nil
-						p.ack = false
-						if s.opts.BodyOnly {
-							p.msg.Body = record.Value
-						} else {
-							if err := s.kopts.Codec.Unmarshal(record.Value, p.msg); err != nil {
-								p.err = err
-								p.msg.Body = record.Value
-								if eh != nil {
-									_ = eh(p)
-									if p.ack {
-										fillOffsets(offsets, record)
-									}
-									pPool.Put(p)
-									continue
-								} else {
-									if s.kopts.Logger.V(logger.ErrorLevel) {
-										s.kopts.Logger.Errorf(s.kopts.Context, "[kgo]: failed to unmarshal: %v", err)
-									}
-								}
-								pPool.Put(p)
-								return err
-							}
-						}
-						err = s.handler(p)
-						if err == nil && s.opts.AutoAck {
-							p.ack = true
-						} else if err != nil {
-							p.err = err
-							if eh != nil {
-								_ = eh(p)
-							} else {
-								if s.kopts.Logger.V(logger.ErrorLevel) {
-									s.kopts.Logger.Errorf(s.kopts.Context, "[kgo]: subscriber error: %v", err)
-								}
-							}
-						}
-						if p.ack {
-							fillOffsets(offsets, record)
-						}
-						pPool.Put(p)
-					}
-					return nil
-				})
-			}
-			if err := g.Wait(); err != nil {
-				return err
-			}
-		}
-	}
-
-	close(doneCh)
-
-	return commitOffsets(s.reader, ctx, offsets)
-}
-
 func (k *kBroker) String() string {
 	return "kgo"
 }
 
-func NewBroker(opts ...broker.Option) broker.Broker {
+func NewBroker(opts ...broker.Option) *kBroker {
 	options := broker.NewOptions(opts...)
+	if options.Codec.String() != "noop" {
+		options.Logger.Infof(options.Context, "broker codec not noop, disable plain kafka headers usage")
+	}
 	kopts := []kgo.Opt{
-		kgo.BatchCompression(kgo.NoCompression()),
+		kgo.DisableIdempotentWrite(),
+		kgo.ProducerBatchCompression(kgo.NoCompression()),
 		kgo.WithLogger(&mlogger{l: options.Logger, ctx: options.Context}),
 		kgo.RetryBackoffFn(
 			func() func(int) time.Duration {

kgo_test.go

@@ -10,11 +10,17 @@ import (
 	"time"
 
 	kg "github.com/twmb/franz-go/pkg/kgo"
-	kgo "github.com/unistack-org/micro-broker-kgo/v3"
+	kgo "go.unistack.org/micro-broker-kgo/v3"
-	jsoncodec "github.com/unistack-org/micro-codec-json/v3"
+	"go.unistack.org/micro/v3/broker"
-	"github.com/unistack-org/micro/v3/broker"
+	"go.unistack.org/micro/v3/logger"
-	"github.com/unistack-org/micro/v3/logger"
+	"go.unistack.org/micro/v3/metadata"
-	"github.com/unistack-org/micro/v3/metadata"
+)
+
+var (
+	msgcnt   = int64(12000000)
+	group    = "38"
+	prefill  = false
+	loglevel = logger.InfoLevel
 )
 
 var bm = &broker.Message{
@@ -27,7 +33,9 @@ func TestPubSub(t *testing.T) {
 		t.Skip()
 	}
 
-	logger.DefaultLogger.Init(logger.WithLevel(logger.TraceLevel), logger.WithCallerSkipCount(3))
+	if err := logger.DefaultLogger.Init(logger.WithLevel(loglevel), logger.WithCallerSkipCount(3)); err != nil {
+		t.Fatal(err)
+	}
 	ctx := context.Background()
 
 	var addrs []string
@@ -38,7 +46,6 @@ func TestPubSub(t *testing.T) {
 	}
 
 	b := kgo.NewBroker(
-		broker.Codec(jsoncodec.NewCodec()),
 		broker.Addrs(addrs...),
 		kgo.CommitInterval(5*time.Second),
 		kgo.Options(kg.ClientID("test"), kg.FetchMaxBytes(10*1024*1024)),
@@ -56,18 +63,17 @@ func TestPubSub(t *testing.T) {
 			t.Fatal(err)
 		}
 	}()
+	if prefill {
+		msgs := make([]*broker.Message, 0, msgcnt)
+		for i := int64(0); i < msgcnt; i++ {
+			msgs = append(msgs, bm)
+		}
 
-	/*
+		if err := b.BatchPublish(ctx, msgs); err != nil {
-			msgs := make([]*broker.Message, 0, 600000)
+			t.Fatal(err)
-			for i := 0; i < 600000; i++ {
+		}
-				msgs = append(msgs, bm)
+		//	t.Skip()
-			}
+	}
-
-			if err := b.BatchPublish(ctx, msgs); err != nil {
-				t.Fatal(err)
-			}
-		t.Skip()
-	*/
 	done := make(chan bool, 1)
 	idx := int64(0)
 	fn := func(msg broker.Event) error {
@@ -78,7 +84,7 @@ func TestPubSub(t *testing.T) {
 
 	sub, err := b.Subscribe(ctx, "test", fn,
 		broker.SubscribeAutoAck(true),
-		broker.SubscribeGroup("test29"),
+		broker.SubscribeGroup(group),
 		broker.SubscribeBodyOnly(true))
 	if err != nil {
 		t.Fatal(err)
@@ -98,7 +104,11 @@ func TestPubSub(t *testing.T) {
 		for {
 			select {
 			case <-pticker.C:
-				fmt.Printf("processed %v\n", atomic.LoadInt64(&idx))
+				if prc := atomic.LoadInt64(&idx); prc == msgcnt {
+					close(done)
+				} else {
+					fmt.Printf("processed %v\n", prc)
+				}
 			case <-ticker.C:
 				close(done)
 			}

metrics.go

@@ -6,7 +6,7 @@ import (
 	"time"
 
 	"github.com/twmb/franz-go/pkg/kgo"
-	"github.com/unistack-org/micro/v3/meter"
+	"go.unistack.org/micro/v3/meter"
 )
 
 type metrics struct {
@@ -40,10 +40,10 @@ const (
 
 	metricBrokerThrottleLatencies = "broker_throttle_latencies"
 
-	metricBrokerProduceBytesCompressed   = "produce_bytes_compressed_total"
+	metricBrokerProduceBytesCompressed   = "broker_produce_bytes_compressed_total"
-	metricBrokerProduceBytesUncompressed = "produce_bytes_uncompressed_total"
+	metricBrokerProduceBytesUncompressed = "broker_produce_bytes_uncompressed_total"
-	metricBrokerFetchBytesUncompressed   = "broker_fetch_bytes_uncompressed_total"
+	metricBrokerFetchBytesCompressed     = "broker_consume_bytes_compressed_total"
-	metricBrokerFetchBytesCompressed     = "broker_fetch_bytes_compressed_total"
+	metricBrokerFetchBytesUncompressed   = "broker_consume_bytes_uncompressed_total"
 
 	metricBrokerGroupErrors = "broker_group_errors_total"
 

options.go

@@ -5,8 +5,9 @@ import (
 	"time"
 
 	kgo "github.com/twmb/franz-go/pkg/kgo"
-	"github.com/unistack-org/micro/v3/broker"
+	"go.unistack.org/micro/v3/broker"
-	"github.com/unistack-org/micro/v3/client"
+	"go.unistack.org/micro/v3/client"
+	"go.unistack.org/micro/v3/server"
 )
 
 // DefaultCommitInterval specifies how fast send commit offsets to kafka
@@ -48,9 +49,48 @@ func Options(opts ...kgo.Opt) broker.Option {
 	}
 }
 
+// SubscribeOptions pass additional options to broker
+func SubscribeOptions(opts ...kgo.Opt) broker.SubscribeOption {
+	return func(o *broker.SubscribeOptions) {
+		if o.Context == nil {
+			o.Context = context.Background()
+		}
+		options, ok := o.Context.Value(optionsKey{}).([]kgo.Opt)
+		if !ok {
+			options = make([]kgo.Opt, 0, len(opts))
+		}
+		options = append(options, opts...)
+		o.Context = context.WithValue(o.Context, optionsKey{}, options)
+	}
+}
+
+// SubscriberOptions pass additional options to broker
+func SubscriberOptions(opts ...kgo.Opt) server.SubscriberOption {
+	return func(o *server.SubscriberOptions) {
+		if o.Context == nil {
+			o.Context = context.Background()
+		}
+		options, ok := o.Context.Value(optionsKey{}).([]kgo.Opt)
+		if !ok {
+			options = make([]kgo.Opt, 0, len(opts))
+		}
+		options = append(options, opts...)
+		o.Context = context.WithValue(o.Context, optionsKey{}, options)
+	}
+}
+
 type commitIntervalKey struct{}
 
 // CommitInterval specifies interval to send commits
 func CommitInterval(td time.Duration) broker.Option {
 	return broker.SetOption(commitIntervalKey{}, td)
 }
+
+var DefaultSubscribeMaxInflight = 1000
+
+type subscribeMaxInflightKey struct{}
+
+// SubscribeMaxInFlight max queued messages
+func SubscribeMaxInFlight(n int) broker.SubscribeOption {
+	return broker.SetSubscribeOption(subscribeMaxInflightKey{}, n)
+}

util.go

@@ -0,0 +1,244 @@
+package kgo
+
+import (
+	"context"
+	"errors"
+	"sync"
+
+	kgo "github.com/twmb/franz-go/pkg/kgo"
+	"go.unistack.org/micro/v3/broker"
+	"go.unistack.org/micro/v3/logger"
+	"go.unistack.org/micro/v3/metadata"
+)
+
+var ErrLostMessage = errors.New("message not marked for offsets commit and will be lost in next iteration")
+
+var pPool = sync.Pool{
+	New: func() interface{} {
+		return &publication{msg: broker.NewMessage("")}
+	},
+}
+
+type worker struct {
+	done         chan struct{}
+	recs         chan []*kgo.Record
+	cherr        chan error
+	handler      broker.Handler
+	batchHandler broker.BatchHandler
+	opts         broker.SubscribeOptions
+	kopts        broker.Options
+	tpmap        map[string][]int32
+	maxInflight  int
+	reader       *kgo.Client
+	ctx          context.Context
+}
+
+func (s *subscriber) run(ctx context.Context) {
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		case <-s.kopts.Context.Done():
+			return
+		default:
+			fetches := s.reader.PollFetches(ctx)
+			if fetches.IsClientClosed() {
+				// TODO: fatal ?
+				return
+			}
+			if len(fetches.Errors()) > 0 {
+				for _, err := range fetches.Errors() {
+					s.kopts.Logger.Fatalf(ctx, "fetch err topic %s partition %d: %v", err.Topic, err.Partition, err.Err)
+				}
+				// TODO: fatal ?
+				return
+			}
+
+			fetches.EachPartition(func(p kgo.FetchTopicPartition) {
+				s.Lock()
+				consumers := s.consumers[p.Topic]
+				s.Unlock()
+				if consumers == nil {
+					return
+				}
+				w, ok := consumers[p.Partition]
+				if !ok {
+					return
+				}
+				select {
+				case err := <-w.cherr:
+					s.kopts.Logger.Fatalf(ctx, "handle err: %v", err)
+					return
+				case w.recs <- p.Records:
+				case <-w.done:
+				}
+			})
+		}
+	}
+}
+
+func (s *subscriber) assigned(ctx context.Context, _ *kgo.Client, assigned map[string][]int32) {
+	maxInflight := DefaultSubscribeMaxInflight
+
+	if s.opts.Context != nil {
+		if n, ok := s.opts.Context.Value(subscribeMaxInflightKey{}).(int); n > 0 && ok {
+			maxInflight = n
+		}
+	}
+
+	s.Lock()
+	for topic, partitions := range assigned {
+		if s.consumers[topic] == nil {
+			s.consumers[topic] = make(map[int32]worker)
+		}
+		for _, partition := range partitions {
+			w := worker{
+				done:         make(chan struct{}),
+				recs:         make(chan []*kgo.Record),
+				cherr:        make(chan error),
+				kopts:        s.kopts,
+				opts:         s.opts,
+				ctx:          ctx,
+				tpmap:        map[string][]int32{topic: []int32{partition}},
+				reader:       s.reader,
+				handler:      s.handler,
+				batchHandler: s.batchhandler,
+				maxInflight:  maxInflight,
+			}
+			s.consumers[topic][partition] = w
+			go w.handle()
+		}
+	}
+	s.Unlock()
+}
+
+func (s *subscriber) revoked(_ context.Context, _ *kgo.Client, revoked map[string][]int32) {
+	s.Lock()
+	for topic, partitions := range revoked {
+		ptopics := s.consumers[topic]
+		for _, partition := range partitions {
+			w := ptopics[partition]
+			delete(ptopics, partition)
+			if len(ptopics) == 0 {
+				delete(s.consumers, topic)
+			}
+			close(w.done)
+		}
+	}
+	s.Unlock()
+}
+
+func (w *worker) handle() {
+	var err error
+
+	eh := w.kopts.ErrorHandler
+	if w.opts.ErrorHandler != nil {
+		eh = w.opts.ErrorHandler
+	}
+
+	paused := false
+	for {
+		select {
+		case <-w.ctx.Done():
+			w.cherr <- w.ctx.Err()
+			return
+		case <-w.done:
+			return
+		case recs := <-w.recs:
+			if len(recs) >= w.maxInflight {
+				paused = true
+				w.reader.PauseFetchPartitions(w.tpmap)
+			}
+			for _, record := range recs {
+				p := pPool.Get().(*publication)
+				p.msg.Header = nil
+				p.msg.Body = nil
+				p.topic = record.Topic
+				p.err = nil
+				p.ack = false
+				if w.opts.BodyOnly {
+					p.msg.Body = record.Value
+					if l := len(record.Headers); l > 0 {
+						if p.msg.Header == nil {
+							p.msg.Header = metadata.New(l)
+						}
+						for _, h := range record.Headers {
+							p.msg.Header.Set(h.Key, string(h.Value))
+						}
+					}
+				} else if w.kopts.Codec.String() == "noop" {
+					p.msg.Body = record.Value
+					p.msg.Header = metadata.New(len(record.Headers))
+					for _, h := range record.Headers {
+						p.msg.Header.Set(h.Key, string(h.Value))
+					}
+				} else {
+					if err := w.kopts.Codec.Unmarshal(record.Value, p.msg); err != nil {
+						p.err = err
+						p.msg.Body = record.Value
+						if l := len(record.Headers); l > 0 {
+							if p.msg.Header == nil {
+								p.msg.Header = metadata.New(l)
+							}
+							for _, h := range record.Headers {
+								p.msg.Header.Set(h.Key, string(h.Value))
+							}
+						}
+						if eh != nil {
+							_ = eh(p)
+							if p.ack {
+								w.reader.MarkCommitRecords(record)
+							} else {
+								w.cherr <- ErrLostMessage
+								pPool.Put(p)
+								return
+							}
+							pPool.Put(p)
+							continue
+						} else {
+							if w.kopts.Logger.V(logger.ErrorLevel) {
+								w.kopts.Logger.Errorf(w.kopts.Context, "[kgo]: failed to unmarshal: %v", err)
+							}
+						}
+						pPool.Put(p)
+						w.cherr <- err
+						return
+					}
+					if l := len(record.Headers); l > 0 {
+						if p.msg.Header == nil {
+							p.msg.Header = metadata.New(l)
+						}
+						for _, h := range record.Headers {
+							p.msg.Header.Set(h.Key, string(h.Value))
+						}
+					}
+				}
+				err = w.handler(p)
+				if err == nil && w.opts.AutoAck {
+					p.ack = true
+				} else if err != nil {
+					p.err = err
+					if eh != nil {
+						_ = eh(p)
+					} else {
+						if w.kopts.Logger.V(logger.ErrorLevel) {
+							w.kopts.Logger.Errorf(w.kopts.Context, "[kgo]: subscriber error: %v", err)
+						}
+					}
+				}
+				if p.ack {
+					pPool.Put(p)
+					w.reader.MarkCommitRecords(record)
+				} else {
+					pPool.Put(p)
+					w.cherr <- ErrLostMessage
+					return
+				}
+			}
+			if paused {
+				paused = false
+				w.reader.ResumeFetchPartitions(w.tpmap)
+			}
+		}
+	}
+}