diff --git a/go.mod b/go.mod index 388b7cc..fa82c1e 100644 --- a/go.mod +++ b/go.mod @@ -4,9 +4,10 @@ go 1.16 require ( github.com/golang/snappy v0.0.4 // indirect - github.com/google/uuid v1.3.0 - github.com/klauspost/compress v1.13.3 // indirect + github.com/klauspost/compress v1.13.4 // indirect github.com/pierrec/lz4/v4 v4.1.8 // indirect - github.com/twmb/franz-go v0.9.0 - github.com/unistack-org/micro/v3 v3.6.1 + github.com/twmb/franz-go v0.10.2-0.20210820215004-4c20135fffc1 + github.com/unistack-org/micro-codec-json/v3 v3.2.5 + github.com/unistack-org/micro/v3 v3.6.3 + golang.org/x/sync v0.0.0-20210220032951-036812b2e83c ) diff --git a/go.sum b/go.sum index f99ee77..d3dbd75 100644 --- a/go.sum +++ b/go.sum @@ -1,18 +1,19 @@ +github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/ef-ds/deque v1.0.4/go.mod h1:gXDnTC3yqvBcHbq2lcExjtAcVrOnJCbMcZXmuj8Z4tg= -github.com/golang-jwt/jwt v3.2.2+incompatible/go.mod h1:8pz2t5EyA70fFQQSrl6XZXzqecmYZeUEB8OUGHkxJ+I= -github.com/golang/snappy v0.0.3 h1:fHPg5GQYlCeLIPB9BZqMVR5nR9A+IM5zcgeTdjMYmLA= +github.com/golang-jwt/jwt/v4 v4.0.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/google/go-cmp v0.3.1 h1:Xye71clBPdm5HgqGwUkwhbynsUJZhDbS20FvLhQ2izg= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= -github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/go-cmp v0.5.5 h1:Khx7svrCpmxxtHBq5j2mp/xVjsi8hQMfNLvJFAlrGgU= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gorilla/securecookie v1.1.1/go.mod h1:ra0sb63/xPlUeL+yeDciTfxMRAA+MP+HVt/4epWDjd4= github.com/gorilla/sessions v1.2.1/go.mod h1:dk2InVEVJ0sfLlnXv9EAgkf6ecYs/i80K/zI+bUmuGM= github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/imdario/mergo v0.3.12 h1:b6R2BslTbIEToALKP7LxUvijTsNI9TAe80pLWN2g/HU= github.com/imdario/mergo v0.3.12/go.mod h1:jmQim1M+e3UYxmgPu/WyfjB3N3VflVyUjjjwH0dnCYA= github.com/jcmturner/aescts/v2 v2.0.0/go.mod h1:AiaICIRyfYg35RUkr8yESTqvSy7csK90qZ5xfvvsoNs= github.com/jcmturner/dnsutils/v2 v2.0.0/go.mod h1:b0TnjGOvI/n42bZa+hmXL+kFJZsFT7G4t3HTlQ184QM= @@ -20,35 +21,36 @@ github.com/jcmturner/gofork v1.0.0/go.mod h1:MK8+TM0La+2rjBD4jE12Kj1pCCxK7d2LK/U github.com/jcmturner/goidentity/v6 v6.0.1/go.mod h1:X1YW3bgtvwAXju7V3LCIMpY0Gbxyjn/mY9zx4tFonSg= github.com/jcmturner/gokrb5/v8 v8.4.2/go.mod h1:sb+Xq/fTY5yktf/VxLsE3wlfPqQjp0aWNYyvBVK62bc= github.com/jcmturner/rpc/v2 v2.0.3/go.mod h1:VUJYCIDm3PVOEHw8sgt091/20OJjskO/YJki3ELg/Hc= -github.com/klauspost/compress v1.13.0 h1:2T7tUoQrQT+fQWdaY5rjWztFGAFwbGD04iPJg90ZiOs= github.com/klauspost/compress v1.13.0/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= -github.com/klauspost/compress v1.13.3 h1:BtAvtV1+h0YwSVwWoYXMREPpYu9VzTJ9QDI1TEg/iQQ= -github.com/klauspost/compress v1.13.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= -github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= +github.com/klauspost/compress v1.13.4 h1:0zhec2I8zGnjWcKyLl6i3gPqKANCCn5e9xmviEEeX6s= +github.com/klauspost/compress v1.13.4/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= -github.com/pierrec/lz4/v4 v4.1.7 h1:UDV9geJWhFIufAliH7HQlz9wP3JA0t748w+RwbWMLow= github.com/pierrec/lz4/v4 v4.1.7/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= github.com/pierrec/lz4/v4 v4.1.8 h1:ieHkV+i2BRzngO4Wd/3HGowuZStgq6QkPsD1eolNAO4= github.com/pierrec/lz4/v4 v4.1.8/go.mod h1:gZWDp/Ze/IJXGXf23ltt2EXimqmTUXEy0GFuRQyBid4= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/silas/dag v0.0.0-20210121180416-41cf55125c34/go.mod h1:7RTUFBdIRC9nZ7/3RyRNH1bdqIShrDejd1YbLwgPS+I= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= -github.com/twmb/franz-go v0.8.6 h1:m49t7tcgUz70hvTpnzHrsvQ/38Q/VKCpEj1FvDetVTE= -github.com/twmb/franz-go v0.8.6/go.mod h1:v6QnB3abhlVAzlIEIO5L/1Emu8NlkreCI2HSps9utH0= -github.com/twmb/franz-go v0.9.0 h1:mI8VvdLeyJh//X8KX9pWOZcp272eR41O8jI7MfGNhFU= -github.com/twmb/franz-go v0.9.0/go.mod h1:Qa6npC7EIi4WoLmnUz9Ue/sPL0k+ex4OyHCYJ2pCAqw= +github.com/stretchr/testify v1.7.0 h1:nwc3DEeHmmLAfoZucVR881uASk0Mfjw8xYJ99tb5CcY= +github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= +github.com/twmb/franz-go v0.10.1 h1:z05pd/6cBy7ywBK9TIjGko1wa2F1zuFgmVPR6uil0ik= +github.com/twmb/franz-go v0.10.1/go.mod h1:Qa6npC7EIi4WoLmnUz9Ue/sPL0k+ex4OyHCYJ2pCAqw= +github.com/twmb/franz-go v0.10.2-0.20210820215004-4c20135fffc1 h1:JNd7gUA9uggKfso+hxeWg0CJbfM8yo124XH8tBv4v/E= +github.com/twmb/franz-go v0.10.2-0.20210820215004-4c20135fffc1/go.mod h1:Qa6npC7EIi4WoLmnUz9Ue/sPL0k+ex4OyHCYJ2pCAqw= github.com/twmb/franz-go/pkg/kmsg v0.0.0-20210726202344-c376dbc9081f h1:Opx7EKsXb4IOPj1ammVNksPFpbXx6aaxdIn4hGjiIpk= github.com/twmb/franz-go/pkg/kmsg v0.0.0-20210726202344-c376dbc9081f/go.mod h1:SxG/xJKhgPu25SamAq0rrucfp7lbzCpEXOC+vH/ELrY= -github.com/twmb/franz-go/pkg/kmsg v0.0.0-20210804073901-278488bb4ea0 h1:mQ6E/Y7o6p5OPWkaDV5Lsy5EBZd2oBWhsTij6I0/Rh4= -github.com/twmb/franz-go/pkg/kmsg v0.0.0-20210804073901-278488bb4ea0/go.mod h1:SxG/xJKhgPu25SamAq0rrucfp7lbzCpEXOC+vH/ELrY= github.com/twmb/go-rbtree v1.0.0 h1:KxN7dXJ8XaZ4cvmHV1qqXTshxX3EBvX/toG5+UR49Mg= github.com/twmb/go-rbtree v1.0.0/go.mod h1:UlIAI8gu3KRPkXSobZnmJfVwCJgEhD/liWzT5ppzIyc= -github.com/unistack-org/micro/v3 v3.6.0 h1:atxcH6C5JWVjXPDQiT8N9SALf1yWaVtpVvvxVdz7Y7s= -github.com/unistack-org/micro/v3 v3.6.0/go.mod h1:zQnZPEy842kQNcyjmVys6tdMjty4PHdyUUKYm1wrg1s= -github.com/unistack-org/micro/v3 v3.6.1 h1:plaRQHatJeI1iaE21rtU7Kl6tbcbK08gQuw0Fj9Bq/4= -github.com/unistack-org/micro/v3 v3.6.1/go.mod h1:zQnZPEy842kQNcyjmVys6tdMjty4PHdyUUKYm1wrg1s= +github.com/unistack-org/micro-codec-json/v3 v3.2.5 h1:WOilhbL0YSu58iIQIIxpawRYZyx6CR16tCpbX4ai3Vc= +github.com/unistack-org/micro-codec-json/v3 v3.2.5/go.mod h1:LSzfrD9GYWCl6KOyihywx1wlbOgStrpyy3NVHNZAvHA= +github.com/unistack-org/micro-proto v0.0.5 h1:DIC97Hufa2nGjuvTsfToD9laEOKddWMRTzeCfBwJ1j8= +github.com/unistack-org/micro-proto v0.0.5/go.mod h1:EuI7UlfGXmT1hy6WacULib9LbNgRnDYQvTCFoLgKM2I= +github.com/unistack-org/micro/v3 v3.3.19/go.mod h1:LXmPfbJnJNvL0kQs8HfnkV3Wya2Wb+C7keVq++RCZnk= +github.com/unistack-org/micro/v3 v3.6.3 h1:CvC4B2kOgwzhPx+w9fcbEIJkgzxS7i84PRB+vL6Vz0U= +github.com/unistack-org/micro/v3 v3.6.3/go.mod h1:Hp+DmX1Bt+/z+ihk5coYwNtXhPMwr4SJuCFRy2kyUl8= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20201112155050-0c6587e931a9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5tA2iutPOUgjI07+tq3H3K9MVA1s8= @@ -57,6 +59,8 @@ golang.org/x/net v0.0.0-20200114155413-6afb5195e5aa/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210510120150-4163338589ed/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= golang.org/x/net v0.0.0-20210525063256-abc453219eb5/go.mod h1:9nx3DQGgdP8bBQD5qxJ1jj9UTztislL4KSBs9R2vV5Y= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ= +golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -66,7 +70,13 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c h1:dUUwHk2QECo/6vqA44rthZ8ie2QXMNeKRTHCNY2nXvo= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= diff --git a/kgo.go b/kgo.go index 41fd3df..6d81ec1 100644 --- a/kgo.go +++ b/kgo.go @@ -4,17 +4,19 @@ package kgo import ( "context" "fmt" + "math/rand" "net" "strings" "sync" "time" - "github.com/google/uuid" kgo "github.com/twmb/franz-go/pkg/kgo" sasl "github.com/twmb/franz-go/pkg/sasl" "github.com/unistack-org/micro/v3/broker" "github.com/unistack-org/micro/v3/logger" "github.com/unistack-org/micro/v3/metadata" + "github.com/unistack-org/micro/v3/util/id" + "golang.org/x/sync/errgroup" ) type kBroker struct { @@ -44,6 +46,7 @@ type publication struct { err error sync.RWMutex msg *broker.Message + ack bool } func (p *publication) Topic() string { @@ -55,6 +58,7 @@ func (p *publication) Message() *broker.Message { } func (p *publication) Ack() error { + p.ack = true return nil } @@ -85,9 +89,6 @@ func (s *subscriber) Unsubscribe(ctx context.Context) error { } func (k *kBroker) Address() string { - if len(k.opts.Addrs) > 0 { - return k.opts.Addrs[0] - } return strings.Join(k.opts.Addrs, ",") } @@ -108,11 +109,21 @@ func (k *kBroker) Connect(ctx context.Context) error { nctx = ctx } + kaddrs := k.opts.Addrs + + // shuffle addrs + rand.Seed(time.Now().UnixNano()) + rand.Shuffle(len(kaddrs), func(i, j int) { + kaddrs[i], kaddrs[j] = kaddrs[j], kaddrs[i] + }) + + kopts := append(k.kopts, kgo.SeedBrokers(kaddrs...)) + select { case <-nctx.Done(): return nctx.Err() default: - c, err := kgo.NewClient(k.kopts...) + c, err := kgo.NewClient(kopts...) if err != nil { return err } @@ -181,7 +192,8 @@ func (k *kBroker) Init(opts ...broker.Option) error { return err } - kopts := append(k.kopts, kgo.SeedBrokers(k.opts.Addrs...)) + var kopts []kgo.Opt + if k.opts.Context != nil { if v, ok := k.opts.Context.Value(optionsKey{}).([]kgo.Opt); ok && len(v) > 0 { kopts = append(kopts, v...) @@ -210,18 +222,18 @@ func (k *kBroker) Init(opts ...broker.Option) error { if v, ok := k.opts.Context.Value(metadataMinAgeKey{}).(time.Duration); ok { kopts = append(kopts, kgo.MetadataMinAge(v)) } - if v, ok := k.opts.Context.Value(produceRetriesKey{}).(int); ok { - kopts = append(kopts, kgo.ProduceRetries(v)) - } + // if v, ok := k.opts.Context.Value(produceRetriesKey{}).(int); ok { + // kopts = append(kopts, kgo.ProduceRetries(v)) + // } if v, ok := k.opts.Context.Value(requestRetriesKey{}).(int); ok { kopts = append(kopts, kgo.RequestRetries(v)) } - if v, ok := k.opts.Context.Value(retryBackoffKey{}).(func(int) time.Duration); ok { - kopts = append(kopts, kgo.RetryBackoff(v)) - } - if v, ok := k.opts.Context.Value(retryTimeoutKey{}).(func(int16) time.Duration); ok { - kopts = append(kopts, kgo.RetryTimeout(v)) - } + /// if v, ok := k.opts.Context.Value(retryBackoffKey{}).(func(int) time.Duration); ok { + // kopts = append(kopts, kgo.RetryBackoff(v)) + // } + // if v, ok := k.opts.Context.Value(retryTimeoutKey{}).(func(int16) time.Duration); ok { + // kopts = append(kopts, kgo.RetryTimeout(v)) + // } if v, ok := k.opts.Context.Value(saslKey{}).([]sasl.Mechanism); ok { kopts = append(kopts, kgo.SASL(v...)) } @@ -229,6 +241,10 @@ func (k *kBroker) Init(opts ...broker.Option) error { kopts = append(kopts, kgo.WithHooks(v...)) } } + kopts = append(kopts, + //kgo.WithLogger(&mlogger{l: k.opts.Logger, ctx: k.opts.Context}), + kgo.RequiredAcks(kgo.AllISRAcks()), + ) k.kopts = kopts k.init = true @@ -252,6 +268,13 @@ func (k *kBroker) Publish(ctx context.Context, topic string, msg *broker.Message func (k *kBroker) publish(ctx context.Context, msgs []*broker.Message, opts ...broker.PublishOption) error { records := make([]*kgo.Record, 0, len(msgs)) var errs []string + + for _, msg := range msgs { + topic, _ := msg.Header.Get(metadata.HeaderTopic) + rec := &kgo.Record{Value: msg.Body, Topic: topic} + records = append(records, rec) + } + results := k.writer.ProduceSync(ctx, records...) for _, result := range results { if result.Err != nil { @@ -261,6 +284,7 @@ func (k *kBroker) publish(ctx context.Context, msgs []*broker.Message, opts ...b if len(errs) > 0 { return fmt.Errorf("publish error: %s", strings.Join(errs, "\n")) } + return nil } @@ -310,19 +334,29 @@ func (k *kBroker) Subscribe(ctx context.Context, topic string, handler broker.Ha options := broker.NewSubscribeOptions(opts...) if options.Group == "" { - uid, err := uuid.NewRandom() + uid, err := id.New() if err != nil { return nil, err } - options.Group = uid.String() + options.Group = uid } + kaddrs := k.opts.Addrs + + // shuffle addrs + rand.Seed(time.Now().UnixNano()) + rand.Shuffle(len(kaddrs), func(i, j int) { + kaddrs[i], kaddrs[j] = kaddrs[j], kaddrs[i] + }) + kopts := append(k.kopts, + kgo.SeedBrokers(kaddrs...), kgo.ConsumerGroup(options.Group), kgo.ConsumeTopics(topic), kgo.ConsumeResetOffset(kgo.NewOffset().AtStart()), kgo.DisableAutoCommit(), - kgo.WithLogger(&mlogger{l: k.opts.Logger, ctx: k.opts.Context}), + kgo.FetchMaxWait(1*time.Second), + // TODO: must set https://pkg.go.dev/github.com/twmb/franz-go/pkg/kgo#OnRevoked ) @@ -348,21 +382,22 @@ func (s *subscriber) run(ctx context.Context) { case <-s.kopts.Context.Done(): return default: + fmt.Printf("poll fetches\n") fetches := s.reader.PollFetches(ctx) + fmt.Printf("fetches polled\n") if fetches.IsClientClosed() { + fmt.Printf("CCCCCC\n") return } fetches.EachError(func(t string, p int32, err error) { s.kopts.Logger.Errorf(ctx, "fetch err topic %s partition %d: %v", t, p, err) }) - s.handleFetches(ctx, fetches) } } } func (s *subscriber) handleFetches(ctx context.Context, fetches kgo.Fetches) error { - var records []*kgo.Record var batch bool var err error @@ -371,59 +406,84 @@ func (s *subscriber) handleFetches(ctx context.Context, fetches kgo.Fetches) err } _ = batch + mprecords := make(map[int32][]*kgo.Record) + + cnt := 0 for _, fetch := range fetches { for _, ftopic := range fetch.Topics { - ridx := 0 - for { - for _, partition := range ftopic.Partitions { - if ridx >= len(partition.Records) { - continue - } - records = append(records, partition.Records[ridx]) - } + for _, partition := range ftopic.Partitions { + mprecords[partition.Partition] = append(mprecords[partition.Partition], partition.Records...) + cnt += len(partition.Records) + } + } + } - for _, record := range records { - eh := s.kopts.ErrorHandler - if s.opts.ErrorHandler != nil { - eh = s.opts.ErrorHandler - } - p := &publication{topic: record.Topic, msg: &broker.Message{}} + // preallocate optimistic + crecords := make([]*kgo.Record, 0, cnt) - if s.opts.BodyOnly { - p.msg.Body = record.Value - } else { - if err := s.kopts.Codec.Unmarshal(record.Value, p.msg); err != nil { - p.err = err - p.msg.Body = record.Value - if eh != nil { - _ = eh(p) - } else { - if s.kopts.Logger.V(logger.ErrorLevel) { - s.kopts.Logger.Errorf(s.kopts.Context, "[kgo]: failed to unmarshal: %v", err) - } - } - continue - } - } - err = s.handler(p) - if err == nil && s.opts.AutoAck { - records = append(records, record) - } else if err != nil { + eh := s.kopts.ErrorHandler + if s.opts.ErrorHandler != nil { + eh = s.opts.ErrorHandler + } + + g := &errgroup.Group{} + + for _, records := range mprecords { + precords := records + g.Go(func() error { + for _, record := range precords { + p := &publication{topic: record.Topic, msg: &broker.Message{}} + if s.opts.BodyOnly { + p.msg.Body = record.Value + } else { + if err := s.kopts.Codec.Unmarshal(record.Value, p.msg); err != nil { p.err = err + p.msg.Body = record.Value if eh != nil { _ = eh(p) + if p.ack { + crecords = append(crecords, record) + } + return nil } else { if s.kopts.Logger.V(logger.ErrorLevel) { - s.kopts.Logger.Errorf(s.kopts.Context, "[kgo]: subscriber error: %v", err) + s.kopts.Logger.Errorf(s.kopts.Context, "[kgo]: failed to unmarshal: %v", err) } } + return err + } + } + err = s.handler(p) + if err == nil && (s.opts.AutoAck || p.ack) { + crecords = append(crecords, record) + } + if err != nil { + p.err = err + if eh != nil { + _ = eh(p) + } else { + if s.kopts.Logger.V(logger.ErrorLevel) { + s.kopts.Logger.Errorf(s.kopts.Context, "[kgo]: subscriber error: %v", err) + } + } + if p.ack { + crecords = append(crecords, record) } } } - ridx++ - } + return nil + }) } - return s.reader.CommitRecords(ctx, records...) + + if err := g.Wait(); err != nil { + return err + } + + if s.kopts.Logger.V(logger.DebugLevel) { + logger.Debugf(ctx, "commit %d records", len(crecords)) + } + + return s.reader.CommitRecords(ctx, crecords...) } func (k *kBroker) String() string { diff --git a/kgo_test.go b/kgo_test.go index 496da98..f53ef36 100644 --- a/kgo_test.go +++ b/kgo_test.go @@ -6,15 +6,18 @@ import ( "os" "strings" "testing" + "time" kgo "github.com/unistack-org/micro-broker-kgo/v3" + jsoncodec "github.com/unistack-org/micro-codec-json/v3" "github.com/unistack-org/micro/v3/broker" "github.com/unistack-org/micro/v3/logger" + "github.com/unistack-org/micro/v3/metadata" ) var ( bm = &broker.Message{ - Header: map[string]string{"hkey": "hval"}, + Header: map[string]string{"hkey": "hval", metadata.HeaderTopic: "test"}, Body: []byte(`"body"`), } ) @@ -24,17 +27,17 @@ func TestPubSub(t *testing.T) { t.Skip() } - logger.DefaultLogger.Init(logger.WithLevel(logger.TraceLevel)) + logger.DefaultLogger.Init(logger.WithLevel(logger.TraceLevel), logger.WithCallerSkipCount(3)) ctx := context.Background() var addrs []string if addr := os.Getenv("BROKER_ADDRS"); len(addr) == 0 { - addrs = []string{"127.0.0.1:9092"} + addrs = []string{"127.0.0.1:29091", "127.0.0.2:29092", "127.0.0.3:29093"} } else { addrs = strings.Split(addr, ",") } - b := kgo.NewBroker(broker.Addrs(addrs...), kgo.ClientID("test")) + b := kgo.NewBroker(broker.Codec(jsoncodec.NewCodec()), broker.Addrs(addrs...), kgo.ClientID("test")) if err := b.Init(); err != nil { t.Fatal(err) } @@ -49,14 +52,26 @@ func TestPubSub(t *testing.T) { } }() + /* + fmt.Printf("prefill") + + msgs := make([]*broker.Message, 0, 600000) + for i := 0; i < 600000; i++ { + msgs = append(msgs, bm) + } + + if err := b.BatchPublish(ctx, msgs); err != nil { + t.Fatal(err) + } + */ done := make(chan bool, 1) + idx := 0 fn := func(msg broker.Event) error { - fmt.Printf("EEEE %s\n", msg.Message().Body) - done <- true + idx++ return msg.Ack() } - sub, err := b.Subscribe(ctx, "test_topic", fn, broker.SubscribeGroup("test")) + sub, err := b.Subscribe(ctx, "test", fn, broker.SubscribeAutoAck(true), broker.SubscribeGroup("test"), broker.SubscribeBodyOnly(true)) if err != nil { t.Fatal(err) } @@ -65,8 +80,10 @@ func TestPubSub(t *testing.T) { t.Fatal(err) } }() - if err := b.Publish(ctx, "test_topic", bm); err != nil { - t.Fatal(err) + + for { + fmt.Printf("processed %v\n", idx) + time.Sleep(1 * time.Second) } <-done }