intermediate fixes
Signed-off-by: Vasiliy Tolstov <v.tolstov@unistack.org>
This commit is contained in:
parent
fc9af8c63e
commit
3f7fcaf4aa
9
go.mod
9
go.mod
@ -4,9 +4,10 @@ go 1.16
|
||||
|
||||
require (
|
||||
github.com/golang/snappy v0.0.4 // indirect
|
||||
github.com/google/uuid v1.3.0
|
||||
github.com/klauspost/compress v1.13.3 // indirect
|
||||
github.com/klauspost/compress v1.13.4 // indirect
|
||||
github.com/pierrec/lz4/v4 v4.1.8 // indirect
|
||||
github.com/twmb/franz-go v0.9.0
|
||||
github.com/unistack-org/micro/v3 v3.6.1
|
||||
github.com/twmb/franz-go v0.10.2-0.20210820215004-4c20135fffc1
|
||||
github.com/unistack-org/micro-codec-json/v3 v3.2.5
|
||||
github.com/unistack-org/micro/v3 v3.6.3
|
||||
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
|
||||
)
|
||||
|
||||
52
go.sum
52
go.sum
@ -1,18 +1,19 @@
|
||||
github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8=
|
||||
github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38=
|
||||
github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ=
|
||||
github.com/ef-ds/deque v1.0.4/go.mod h1:gXDnTC3yqvBcHbq2lcExjtAcVrOnJCbMcZXmuj8Z4tg=
|
||||
github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I=
|
||||
github.com/golang/snappy v0.0.3 h1:fHPg5GQYlCeLIPB9BZqMVR5nR9A+IM5zcgeTdjMYmLA=
|
||||
github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg=
|
||||
github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk=
|
||||
github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
|
||||
github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM=
|
||||
github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q=
|
||||
github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg=
|
||||
github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU=
|
||||
github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I=
|
||||
github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU=
|
||||
github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
|
||||
github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
|
||||
github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4=
|
||||
github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM=
|
||||
github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
|
||||
github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU=
|
||||
github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA=
|
||||
github.com/jcmturner/aescts/v2 v2.0.0/go.mod h1:AiaICIRyfYg35RUkr8yESTqvSy7csK90qZ5xfvvsoNs=
|
||||
github.com/jcmturner/dnsutils/v2 v2.0.0/go.mod h1:b0TnjGOvI/n42bZa+hmXL+kFJZsFT7G4t3HTlQ184QM=
|
||||
@ -20,35 +21,36 @@ github.com/jcmturner/gofork v1.0.0/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/U
|
||||
github.com/jcmturner/goidentity/v6 v6.0.1/go.mod h1:X1YW3bgtvwAXju7V3LCIMpY0Gbxyjn/mY9zx4tFonSg=
|
||||
github.com/jcmturner/gokrb5/v8 v8.4.2/go.mod h1:sb+Xq/fTY5yktf/VxLsE3wlfPqQjp0aWNYyvBVK62bc=
|
||||
github.com/jcmturner/rpc/v2 v2.0.3/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc=
|
||||
github.com/klauspost/compress v1.13.0 h1:2T7tUoQrQT+fQWdaY5rjWztFGAFwbGD04iPJg90ZiOs=
|
||||
github.com/klauspost/compress v1.13.0/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg=
|
||||
github.com/klauspost/compress v1.13.3 h1:BtAvtV1+h0YwSVwWoYXMREPpYu9VzTJ9QDI1TEg/iQQ=
|
||||
github.com/klauspost/compress v1.13.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg=
|
||||
github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc=
|
||||
github.com/klauspost/compress v1.13.4 h1:0zhec2I8zGnjWcKyLl6i3gPqKANCCn5e9xmviEEeX6s=
|
||||
github.com/klauspost/compress v1.13.4/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg=
|
||||
github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
|
||||
github.com/pierrec/lz4/v4 v4.1.7 h1:UDV9geJWhFIufAliH7HQlz9wP3JA0t748w+RwbWMLow=
|
||||
github.com/pierrec/lz4/v4 v4.1.7/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
|
||||
github.com/pierrec/lz4/v4 v4.1.8 h1:ieHkV+i2BRzngO4Wd/3HGowuZStgq6QkPsD1eolNAO4=
|
||||
github.com/pierrec/lz4/v4 v4.1.8/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4=
|
||||
github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
|
||||
github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
|
||||
github.com/silas/dag v0.0.0-20210121180416-41cf55125c34/go.mod h1:7RTUFBdIRC9nZ7/3RyRNH1bdqIShrDejd1YbLwgPS+I=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
|
||||
github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/twmb/franz-go v0.8.6 h1:m49t7tcgUz70hvTpnzHrsvQ/38Q/VKCpEj1FvDetVTE=
|
||||
github.com/twmb/franz-go v0.8.6/go.mod h1:v6QnB3abhlVAzlIEIO5L/1Emu8NlkreCI2HSps9utH0=
|
||||
github.com/twmb/franz-go v0.9.0 h1:mI8VvdLeyJh//X8KX9pWOZcp272eR41O8jI7MfGNhFU=
|
||||
github.com/twmb/franz-go v0.9.0/go.mod h1:Qa6npC7EIi4WoLmnUz9Ue/sPL0k+ex4OyHCYJ2pCAqw=
|
||||
github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY=
|
||||
github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg=
|
||||
github.com/twmb/franz-go v0.10.1 h1:z05pd/6cBy7ywBK9TIjGko1wa2F1zuFgmVPR6uil0ik=
|
||||
github.com/twmb/franz-go v0.10.1/go.mod h1:Qa6npC7EIi4WoLmnUz9Ue/sPL0k+ex4OyHCYJ2pCAqw=
|
||||
github.com/twmb/franz-go v0.10.2-0.20210820215004-4c20135fffc1 h1:JNd7gUA9uggKfso+hxeWg0CJbfM8yo124XH8tBv4v/E=
|
||||
github.com/twmb/franz-go v0.10.2-0.20210820215004-4c20135fffc1/go.mod h1:Qa6npC7EIi4WoLmnUz9Ue/sPL0k+ex4OyHCYJ2pCAqw=
|
||||
github.com/twmb/franz-go/pkg/kmsg v0.0.0-20210726202344-c376dbc9081f h1:Opx7EKsXb4IOPj1ammVNksPFpbXx6aaxdIn4hGjiIpk=
|
||||
github.com/twmb/franz-go/pkg/kmsg v0.0.0-20210726202344-c376dbc9081f/go.mod h1:SxG/xJKhgPu25SamAq0rrucfp7lbzCpEXOC+vH/ELrY=
|
||||
github.com/twmb/franz-go/pkg/kmsg v0.0.0-20210804073901-278488bb4ea0 h1:mQ6E/Y7o6p5OPWkaDV5Lsy5EBZd2oBWhsTij6I0/Rh4=
|
||||
github.com/twmb/franz-go/pkg/kmsg v0.0.0-20210804073901-278488bb4ea0/go.mod h1:SxG/xJKhgPu25SamAq0rrucfp7lbzCpEXOC+vH/ELrY=
|
||||
github.com/twmb/go-rbtree v1.0.0 h1:KxN7dXJ8XaZ4cvmHV1qqXTshxX3EBvX/toG5+UR49Mg=
|
||||
github.com/twmb/go-rbtree v1.0.0/go.mod h1:UlIAI8gu3KRPkXSobZnmJfVwCJgEhD/liWzT5ppzIyc=
|
||||
github.com/unistack-org/micro/v3 v3.6.0 h1:atxcH6C5JWVjXPDQiT8N9SALf1yWaVtpVvvxVdz7Y7s=
|
||||
github.com/unistack-org/micro/v3 v3.6.0/go.mod h1:zQnZPEy842kQNcyjmVys6tdMjty4PHdyUUKYm1wrg1s=
|
||||
github.com/unistack-org/micro/v3 v3.6.1 h1:plaRQHatJeI1iaE21rtU7Kl6tbcbK08gQuw0Fj9Bq/4=
|
||||
github.com/unistack-org/micro/v3 v3.6.1/go.mod h1:zQnZPEy842kQNcyjmVys6tdMjty4PHdyUUKYm1wrg1s=
|
||||
github.com/unistack-org/micro-codec-json/v3 v3.2.5 h1:WOilhbL0YSu58iIQIIxpawRYZyx6CR16tCpbX4ai3Vc=
|
||||
github.com/unistack-org/micro-codec-json/v3 v3.2.5/go.mod h1:LSzfrD9GYWCl6KOyihywx1wlbOgStrpyy3NVHNZAvHA=
|
||||
github.com/unistack-org/micro-proto v0.0.5 h1:DIC97Hufa2nGjuvTsfToD9laEOKddWMRTzeCfBwJ1j8=
|
||||
github.com/unistack-org/micro-proto v0.0.5/go.mod h1:EuI7UlfGXmT1hy6WacULib9LbNgRnDYQvTCFoLgKM2I=
|
||||
github.com/unistack-org/micro/v3 v3.3.19/go.mod h1:LXmPfbJnJNvL0kQs8HfnkV3Wya2Wb+C7keVq++RCZnk=
|
||||
github.com/unistack-org/micro/v3 v3.6.3 h1:CvC4B2kOgwzhPx+w9fcbEIJkgzxS7i84PRB+vL6Vz0U=
|
||||
github.com/unistack-org/micro/v3 v3.6.3/go.mod h1:Hp+DmX1Bt+/z+ihk5coYwNtXhPMwr4SJuCFRy2kyUl8=
|
||||
golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w=
|
||||
golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto=
|
||||
golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8=
|
||||
@ -57,6 +59,8 @@ golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLL
|
||||
golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
|
||||
golang.org/x/net v0.0.0-20210510120150-4163338589ed/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y=
|
||||
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
|
||||
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
|
||||
golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
|
||||
golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
@ -66,7 +70,13 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4=
|
||||
golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0=
|
||||
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
|
||||
google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk=
|
||||
google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
|
||||
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
|
||||
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo=
|
||||
gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
|
||||
|
||||
126
kgo.go
126
kgo.go
@ -4,17 +4,19 @@ package kgo
|
||||
import (
|
||||
"context"
|
||||
"fmt"
|
||||
"math/rand"
|
||||
"net"
|
||||
"strings"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/google/uuid"
|
||||
kgo "github.com/twmb/franz-go/pkg/kgo"
|
||||
sasl "github.com/twmb/franz-go/pkg/sasl"
|
||||
"github.com/unistack-org/micro/v3/broker"
|
||||
"github.com/unistack-org/micro/v3/logger"
|
||||
"github.com/unistack-org/micro/v3/metadata"
|
||||
"github.com/unistack-org/micro/v3/util/id"
|
||||
"golang.org/x/sync/errgroup"
|
||||
)
|
||||
|
||||
type kBroker struct {
|
||||
@ -44,6 +46,7 @@ type publication struct {
|
||||
err error
|
||||
sync.RWMutex
|
||||
msg *broker.Message
|
||||
ack bool
|
||||
}
|
||||
|
||||
func (p *publication) Topic() string {
|
||||
@ -55,6 +58,7 @@ func (p *publication) Message() *broker.Message {
|
||||
}
|
||||
|
||||
func (p *publication) Ack() error {
|
||||
p.ack = true
|
||||
return nil
|
||||
}
|
||||
|
||||
@ -85,9 +89,6 @@ func (s *subscriber) Unsubscribe(ctx context.Context) error {
|
||||
}
|
||||
|
||||
func (k *kBroker) Address() string {
|
||||
if len(k.opts.Addrs) > 0 {
|
||||
return k.opts.Addrs[0]
|
||||
}
|
||||
return strings.Join(k.opts.Addrs, ",")
|
||||
}
|
||||
|
||||
@ -108,11 +109,21 @@ func (k *kBroker) Connect(ctx context.Context) error {
|
||||
nctx = ctx
|
||||
}
|
||||
|
||||
kaddrs := k.opts.Addrs
|
||||
|
||||
// shuffle addrs
|
||||
rand.Seed(time.Now().UnixNano())
|
||||
rand.Shuffle(len(kaddrs), func(i, j int) {
|
||||
kaddrs[i], kaddrs[j] = kaddrs[j], kaddrs[i]
|
||||
})
|
||||
|
||||
kopts := append(k.kopts, kgo.SeedBrokers(kaddrs...))
|
||||
|
||||
select {
|
||||
case <-nctx.Done():
|
||||
return nctx.Err()
|
||||
default:
|
||||
c, err := kgo.NewClient(k.kopts...)
|
||||
c, err := kgo.NewClient(kopts...)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
@ -181,7 +192,8 @@ func (k *kBroker) Init(opts ...broker.Option) error {
|
||||
return err
|
||||
}
|
||||
|
||||
kopts := append(k.kopts, kgo.SeedBrokers(k.opts.Addrs...))
|
||||
var kopts []kgo.Opt
|
||||
|
||||
if k.opts.Context != nil {
|
||||
if v, ok := k.opts.Context.Value(optionsKey{}).([]kgo.Opt); ok && len(v) > 0 {
|
||||
kopts = append(kopts, v...)
|
||||
@ -210,18 +222,18 @@ func (k *kBroker) Init(opts ...broker.Option) error {
|
||||
if v, ok := k.opts.Context.Value(metadataMinAgeKey{}).(time.Duration); ok {
|
||||
kopts = append(kopts, kgo.MetadataMinAge(v))
|
||||
}
|
||||
if v, ok := k.opts.Context.Value(produceRetriesKey{}).(int); ok {
|
||||
kopts = append(kopts, kgo.ProduceRetries(v))
|
||||
}
|
||||
// if v, ok := k.opts.Context.Value(produceRetriesKey{}).(int); ok {
|
||||
// kopts = append(kopts, kgo.ProduceRetries(v))
|
||||
// }
|
||||
if v, ok := k.opts.Context.Value(requestRetriesKey{}).(int); ok {
|
||||
kopts = append(kopts, kgo.RequestRetries(v))
|
||||
}
|
||||
if v, ok := k.opts.Context.Value(retryBackoffKey{}).(func(int) time.Duration); ok {
|
||||
kopts = append(kopts, kgo.RetryBackoff(v))
|
||||
}
|
||||
if v, ok := k.opts.Context.Value(retryTimeoutKey{}).(func(int16) time.Duration); ok {
|
||||
kopts = append(kopts, kgo.RetryTimeout(v))
|
||||
}
|
||||
/// if v, ok := k.opts.Context.Value(retryBackoffKey{}).(func(int) time.Duration); ok {
|
||||
// kopts = append(kopts, kgo.RetryBackoff(v))
|
||||
// }
|
||||
// if v, ok := k.opts.Context.Value(retryTimeoutKey{}).(func(int16) time.Duration); ok {
|
||||
// kopts = append(kopts, kgo.RetryTimeout(v))
|
||||
// }
|
||||
if v, ok := k.opts.Context.Value(saslKey{}).([]sasl.Mechanism); ok {
|
||||
kopts = append(kopts, kgo.SASL(v...))
|
||||
}
|
||||
@ -229,6 +241,10 @@ func (k *kBroker) Init(opts ...broker.Option) error {
|
||||
kopts = append(kopts, kgo.WithHooks(v...))
|
||||
}
|
||||
}
|
||||
kopts = append(kopts,
|
||||
//kgo.WithLogger(&mlogger{l: k.opts.Logger, ctx: k.opts.Context}),
|
||||
kgo.RequiredAcks(kgo.AllISRAcks()),
|
||||
)
|
||||
|
||||
k.kopts = kopts
|
||||
k.init = true
|
||||
@ -252,6 +268,13 @@ func (k *kBroker) Publish(ctx context.Context, topic string, msg *broker.Message
|
||||
func (k *kBroker) publish(ctx context.Context, msgs []*broker.Message, opts ...broker.PublishOption) error {
|
||||
records := make([]*kgo.Record, 0, len(msgs))
|
||||
var errs []string
|
||||
|
||||
for _, msg := range msgs {
|
||||
topic, _ := msg.Header.Get(metadata.HeaderTopic)
|
||||
rec := &kgo.Record{Value: msg.Body, Topic: topic}
|
||||
records = append(records, rec)
|
||||
}
|
||||
|
||||
results := k.writer.ProduceSync(ctx, records...)
|
||||
for _, result := range results {
|
||||
if result.Err != nil {
|
||||
@ -261,6 +284,7 @@ func (k *kBroker) publish(ctx context.Context, msgs []*broker.Message, opts ...b
|
||||
if len(errs) > 0 {
|
||||
return fmt.Errorf("publish error: %s", strings.Join(errs, "\n"))
|
||||
}
|
||||
|
||||
return nil
|
||||
}
|
||||
|
||||
@ -310,19 +334,29 @@ func (k *kBroker) Subscribe(ctx context.Context, topic string, handler broker.Ha
|
||||
options := broker.NewSubscribeOptions(opts...)
|
||||
|
||||
if options.Group == "" {
|
||||
uid, err := uuid.NewRandom()
|
||||
uid, err := id.New()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
options.Group = uid.String()
|
||||
options.Group = uid
|
||||
}
|
||||
|
||||
kaddrs := k.opts.Addrs
|
||||
|
||||
// shuffle addrs
|
||||
rand.Seed(time.Now().UnixNano())
|
||||
rand.Shuffle(len(kaddrs), func(i, j int) {
|
||||
kaddrs[i], kaddrs[j] = kaddrs[j], kaddrs[i]
|
||||
})
|
||||
|
||||
kopts := append(k.kopts,
|
||||
kgo.SeedBrokers(kaddrs...),
|
||||
kgo.ConsumerGroup(options.Group),
|
||||
kgo.ConsumeTopics(topic),
|
||||
kgo.ConsumeResetOffset(kgo.NewOffset().AtStart()),
|
||||
kgo.DisableAutoCommit(),
|
||||
kgo.WithLogger(&mlogger{l: k.opts.Logger, ctx: k.opts.Context}),
|
||||
kgo.FetchMaxWait(1*time.Second),
|
||||
|
||||
// TODO: must set https://pkg.go.dev/github.com/twmb/franz-go/pkg/kgo#OnRevoked
|
||||
)
|
||||
|
||||
@ -348,21 +382,22 @@ func (s *subscriber) run(ctx context.Context) {
|
||||
case <-s.kopts.Context.Done():
|
||||
return
|
||||
default:
|
||||
fmt.Printf("poll fetches\n")
|
||||
fetches := s.reader.PollFetches(ctx)
|
||||
fmt.Printf("fetches polled\n")
|
||||
if fetches.IsClientClosed() {
|
||||
fmt.Printf("CCCCCC\n")
|
||||
return
|
||||
}
|
||||
fetches.EachError(func(t string, p int32, err error) {
|
||||
s.kopts.Logger.Errorf(ctx, "fetch err topic %s partition %d: %v", t, p, err)
|
||||
})
|
||||
|
||||
s.handleFetches(ctx, fetches)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func (s *subscriber) handleFetches(ctx context.Context, fetches kgo.Fetches) error {
|
||||
var records []*kgo.Record
|
||||
var batch bool
|
||||
var err error
|
||||
|
||||
@ -371,24 +406,33 @@ func (s *subscriber) handleFetches(ctx context.Context, fetches kgo.Fetches) err
|
||||
}
|
||||
_ = batch
|
||||
|
||||
mprecords := make(map[int32][]*kgo.Record)
|
||||
|
||||
cnt := 0
|
||||
for _, fetch := range fetches {
|
||||
for _, ftopic := range fetch.Topics {
|
||||
ridx := 0
|
||||
for {
|
||||
for _, partition := range ftopic.Partitions {
|
||||
if ridx >= len(partition.Records) {
|
||||
continue
|
||||
mprecords[partition.Partition] = append(mprecords[partition.Partition], partition.Records...)
|
||||
cnt += len(partition.Records)
|
||||
}
|
||||
}
|
||||
records = append(records, partition.Records[ridx])
|
||||
}
|
||||
|
||||
for _, record := range records {
|
||||
// preallocate optimistic
|
||||
crecords := make([]*kgo.Record, 0, cnt)
|
||||
|
||||
eh := s.kopts.ErrorHandler
|
||||
if s.opts.ErrorHandler != nil {
|
||||
eh = s.opts.ErrorHandler
|
||||
}
|
||||
p := &publication{topic: record.Topic, msg: &broker.Message{}}
|
||||
|
||||
g := &errgroup.Group{}
|
||||
|
||||
for _, records := range mprecords {
|
||||
precords := records
|
||||
g.Go(func() error {
|
||||
for _, record := range precords {
|
||||
p := &publication{topic: record.Topic, msg: &broker.Message{}}
|
||||
if s.opts.BodyOnly {
|
||||
p.msg.Body = record.Value
|
||||
} else {
|
||||
@ -397,18 +441,23 @@ func (s *subscriber) handleFetches(ctx context.Context, fetches kgo.Fetches) err
|
||||
p.msg.Body = record.Value
|
||||
if eh != nil {
|
||||
_ = eh(p)
|
||||
if p.ack {
|
||||
crecords = append(crecords, record)
|
||||
}
|
||||
return nil
|
||||
} else {
|
||||
if s.kopts.Logger.V(logger.ErrorLevel) {
|
||||
s.kopts.Logger.Errorf(s.kopts.Context, "[kgo]: failed to unmarshal: %v", err)
|
||||
}
|
||||
}
|
||||
continue
|
||||
return err
|
||||
}
|
||||
}
|
||||
err = s.handler(p)
|
||||
if err == nil && s.opts.AutoAck {
|
||||
records = append(records, record)
|
||||
} else if err != nil {
|
||||
if err == nil && (s.opts.AutoAck || p.ack) {
|
||||
crecords = append(crecords, record)
|
||||
}
|
||||
if err != nil {
|
||||
p.err = err
|
||||
if eh != nil {
|
||||
_ = eh(p)
|
||||
@ -417,13 +466,24 @@ func (s *subscriber) handleFetches(ctx context.Context, fetches kgo.Fetches) err
|
||||
s.kopts.Logger.Errorf(s.kopts.Context, "[kgo]: subscriber error: %v", err)
|
||||
}
|
||||
}
|
||||
if p.ack {
|
||||
crecords = append(crecords, record)
|
||||
}
|
||||
}
|
||||
}
|
||||
ridx++
|
||||
return nil
|
||||
})
|
||||
}
|
||||
|
||||
if err := g.Wait(); err != nil {
|
||||
return err
|
||||
}
|
||||
return s.reader.CommitRecords(ctx, records...)
|
||||
|
||||
if s.kopts.Logger.V(logger.DebugLevel) {
|
||||
logger.Debugf(ctx, "commit %d records", len(crecords))
|
||||
}
|
||||
|
||||
return s.reader.CommitRecords(ctx, crecords...)
|
||||
}
|
||||
|
||||
func (k *kBroker) String() string {
|
||||
|
||||
35
kgo_test.go
35
kgo_test.go
@ -6,15 +6,18 @@ import (
|
||||
"os"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
kgo "github.com/unistack-org/micro-broker-kgo/v3"
|
||||
jsoncodec "github.com/unistack-org/micro-codec-json/v3"
|
||||
"github.com/unistack-org/micro/v3/broker"
|
||||
"github.com/unistack-org/micro/v3/logger"
|
||||
"github.com/unistack-org/micro/v3/metadata"
|
||||
)
|
||||
|
||||
var (
|
||||
bm = &broker.Message{
|
||||
Header: map[string]string{"hkey": "hval"},
|
||||
Header: map[string]string{"hkey": "hval", metadata.HeaderTopic: "test"},
|
||||
Body: []byte(`"body"`),
|
||||
}
|
||||
)
|
||||
@ -24,17 +27,17 @@ func TestPubSub(t *testing.T) {
|
||||
t.Skip()
|
||||
}
|
||||
|
||||
logger.DefaultLogger.Init(logger.WithLevel(logger.TraceLevel))
|
||||
logger.DefaultLogger.Init(logger.WithLevel(logger.TraceLevel), logger.WithCallerSkipCount(3))
|
||||
ctx := context.Background()
|
||||
|
||||
var addrs []string
|
||||
if addr := os.Getenv("BROKER_ADDRS"); len(addr) == 0 {
|
||||
addrs = []string{"127.0.0.1:9092"}
|
||||
addrs = []string{"127.0.0.1:29091", "127.0.0.2:29092", "127.0.0.3:29093"}
|
||||
} else {
|
||||
addrs = strings.Split(addr, ",")
|
||||
}
|
||||
|
||||
b := kgo.NewBroker(broker.Addrs(addrs...), kgo.ClientID("test"))
|
||||
b := kgo.NewBroker(broker.Codec(jsoncodec.NewCodec()), broker.Addrs(addrs...), kgo.ClientID("test"))
|
||||
if err := b.Init(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@ -49,14 +52,26 @@ func TestPubSub(t *testing.T) {
|
||||
}
|
||||
}()
|
||||
|
||||
/*
|
||||
fmt.Printf("prefill")
|
||||
|
||||
msgs := make([]*broker.Message, 0, 600000)
|
||||
for i := 0; i < 600000; i++ {
|
||||
msgs = append(msgs, bm)
|
||||
}
|
||||
|
||||
if err := b.BatchPublish(ctx, msgs); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
*/
|
||||
done := make(chan bool, 1)
|
||||
idx := 0
|
||||
fn := func(msg broker.Event) error {
|
||||
fmt.Printf("EEEE %s\n", msg.Message().Body)
|
||||
done <- true
|
||||
idx++
|
||||
return msg.Ack()
|
||||
}
|
||||
|
||||
sub, err := b.Subscribe(ctx, "test_topic", fn, broker.SubscribeGroup("test"))
|
||||
sub, err := b.Subscribe(ctx, "test", fn, broker.SubscribeAutoAck(true), broker.SubscribeGroup("test"), broker.SubscribeBodyOnly(true))
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
@ -65,8 +80,10 @@ func TestPubSub(t *testing.T) {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}()
|
||||
if err := b.Publish(ctx, "test_topic", bm); err != nil {
|
||||
t.Fatal(err)
|
||||
|
||||
for {
|
||||
fmt.Printf("processed %v\n", idx)
|
||||
time.Sleep(1 * time.Second)
|
||||
}
|
||||
<-done
|
||||
}
|
||||
|
||||
Loading…
Reference in New Issue
Block a user