unistack-org/micro-broker-kgo
4
0
Fork 2
Code Issues 3 Pull Requests 1 Actions Packages Projects Releases 44 Wiki Activity

cpu and memory optimizations

Browse Source 
Signed-off-by: Vasiliy Tolstov <v.tolstov@unistack.org>
This commit is contained in:
Василий Толстов 2021-08-27 02:24:43 +03:00
parent fe6d7d340e
commit 54ecafc4b1
4 changed files with 131 additions and 97 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

3
go.mod
View File

@ -4,9 +4,10 @@ go 1.16
require (
github.com/twmb/franz-go v0.10.3-0.20210825060253-d5e80b38ca2b
github.com/twmb/franz-go/pkg/kmsg v0.0.0-20210825163214-e185676761dd // indirect
github.com/twmb/franz-go/pkg/kmsg v0.0.0-20210826221812-c6df11da978a
github.com/unistack-org/micro-codec-json/v3 v3.2.5
github.com/unistack-org/micro-proto v0.0.8 // indirect
github.com/unistack-org/micro/v3 v3.6.3
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
google.golang.org/protobuf v1.27.1 // indirect
)

5
go.sum
View File

@ -61,6 +61,8 @@ github.com/twmb/franz-go v0.10.3-0.20210825060253-d5e80b38ca2b/go.mod h1:Txc5/v0
github.com/twmb/franz-go/pkg/kmsg v0.0.0-20210823212011-0d01f7456b4d/go.mod h1:SxG/xJKhgPu25SamAq0rrucfp7lbzCpEXOC+vH/ELrY=
github.com/twmb/franz-go/pkg/kmsg v0.0.0-20210825163214-e185676761dd h1:8TvJbg0dnOaq4Oex1wnYejPokyUQ4zG2OQUfD76yq4M=
github.com/twmb/franz-go/pkg/kmsg v0.0.0-20210825163214-e185676761dd/go.mod h1:SxG/xJKhgPu25SamAq0rrucfp7lbzCpEXOC+vH/ELrY=
github.com/twmb/franz-go/pkg/kmsg v0.0.0-20210826221812-c6df11da978a h1:UBO58MRI/GH5eD5+fKXZoJMGyEcsUs9LQnHJbNL0yKs=
github.com/twmb/franz-go/pkg/kmsg v0.0.0-20210826221812-c6df11da978a/go.mod h1:SxG/xJKhgPu25SamAq0rrucfp7lbzCpEXOC+vH/ELrY=
github.com/twmb/go-rbtree v1.0.0 h1:KxN7dXJ8XaZ4cvmHV1qqXTshxX3EBvX/toG5+UR49Mg=
github.com/twmb/go-rbtree v1.0.0/go.mod h1:UlIAI8gu3KRPkXSobZnmJfVwCJgEhD/liWzT5ppzIyc=
github.com/unistack-org/micro-codec-json/v3 v3.2.5 h1:WOilhbL0YSu58iIQIIxpawRYZyx6CR16tCpbX4ai3Vc=
@ -90,6 +92,7 @@ golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qx
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU=
golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@ -130,6 +133,8 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba
google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk=
google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ=
google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

209
kgo.go
View File

@ -10,7 +10,9 @@ import (
"sync/atomic"
"time"
kerr "github.com/twmb/franz-go/pkg/kerr"
kgo "github.com/twmb/franz-go/pkg/kgo"
kmsg "github.com/twmb/franz-go/pkg/kmsg"
"github.com/unistack-org/micro/v3/broker"
"github.com/unistack-org/micro/v3/logger"
"github.com/unistack-org/micro/v3/metadata"
@ -21,7 +23,7 @@ import (
var pPool = sync.Pool{
New: func() interface{} {
return &broker.Message{}
return &publication{msg: &broker.Message{}}
},
}
@ -336,7 +338,7 @@ func (k *kBroker) Subscribe(ctx context.Context, topic string, handler broker.Ha
kgo.ConsumeResetOffset(kgo.NewOffset().AtStart()),
kgo.DisableAutoCommit(),
kgo.FetchMaxWait(1*time.Second),
kgo.KeepControlRecords(),
// kgo.KeepControlRecords(),
kgo.Balancers(kgo.CooperativeStickyBalancer(), kgo.StickyBalancer()),
kgo.FetchIsolationLevel(kgo.ReadUncommitted()),
// kgo.WithHooks(&metrics{meter: k.opts.Meter}),
@ -348,7 +350,7 @@ func (k *kBroker) Subscribe(ctx context.Context, topic string, handler broker.Ha
return nil, err
}
sub := &subscriber{done: make(chan struct{}), opts: options, reader: reader, handler: handler, kopts: k.opts}
sub := &subscriber{topic: topic, done: make(chan struct{}), opts: options, reader: reader, handler: handler, kopts: k.opts}
go sub.run(ctx)
k.Lock()
@ -390,21 +392,6 @@ func (s *subscriber) run(ctx context.Context) {
func (s *subscriber) handleFetches(ctx context.Context, fetches kgo.Fetches) error {
var err error
mprecords := make(map[int32][]*kgo.Record)
cnt := int64(0)
for _, fetch := range fetches {
for _, ftopic := range fetch.Topics {
for _, partition := range ftopic.Partitions {
mprecords[partition.Partition] = append(mprecords[partition.Partition], partition.Records...)
cnt += int64(len(partition.Records))
}
}
}
// preallocate optimistic
crecords := make([]*kgo.Record, 0, 10000)
eh := s.kopts.ErrorHandler
if s.opts.ErrorHandler != nil {
eh = s.opts.ErrorHandler
@ -412,6 +399,8 @@ func (s *subscriber) handleFetches(ctx context.Context, fetches kgo.Fetches) err
var mu sync.Mutex
done := int32(0)
doneCh := make(chan struct{})
g, gctx := errgroup.WithContext(ctx)
td := DefaultCommitInterval
@ -421,107 +410,142 @@ func (s *subscriber) handleFetches(ctx context.Context, fetches kgo.Fetches) err
}
}
// ticker for commit offsets
ticker := time.NewTicker(td)
defer ticker.Stop()
g.Go(func() error {
offsets := make(map[string]map[int32]kgo.EpochOffset)
offsets[s.topic] = make(map[int32]kgo.EpochOffset)
fillOffsets := func(off map[string]map[int32]kgo.EpochOffset, rec *kgo.Record) {
mu.Lock()
if at, ok := off[s.topic][rec.Partition]; ok {
if at.Epoch > rec.LeaderEpoch || at.Epoch == rec.LeaderEpoch && at.Offset > rec.Offset {
mu.Unlock()
return
}
}
off[s.topic][rec.Partition] = kgo.EpochOffset{Epoch: rec.LeaderEpoch, Offset: rec.Offset + 1}
mu.Unlock()
}
commitOffsets := func(cl *kgo.Client, ctx context.Context, off map[string]map[int32]kgo.EpochOffset) error {
var rerr error
mu.Lock()
offsets := off
mu.Unlock()
cl.CommitOffsetsSync(ctx, offsets, func(_ *kgo.Client, _ *kmsg.OffsetCommitRequest, resp *kmsg.OffsetCommitResponse, err error) {
if err != nil {
rerr = err
return
}
for _, topic := range resp.Topics {
for _, partition := range topic.Partitions {
if err := kerr.ErrorForCode(partition.ErrorCode); err != nil {
rerr = err
return
}
}
}
})
return rerr
}
go func() {
for {
select {
case <-gctx.Done():
return gctx.Err()
return
case <-s.done:
mu.Lock()
err := s.reader.CommitRecords(ctx, crecords...)
mu.Unlock()
return err
atomic.StoreInt32(&done, 1)
if err := commitOffsets(s.reader, ctx, offsets); err != nil && s.kopts.Logger.V(logger.ErrorLevel) {
s.kopts.Logger.Errorf(s.kopts.Context, "[kgo]: failed to commit offsets: %v", err)
}
return
case <-doneCh:
return
case <-ticker.C:
v := atomic.LoadInt64(&cnt)
if v == 0 {
return nil
if err := commitOffsets(s.reader, ctx, offsets); err != nil {
if s.kopts.Logger.V(logger.ErrorLevel) {
s.kopts.Logger.Errorf(s.kopts.Context, "[kgo]: failed to commit offsets: %v", err)
}
return
}
mu.Lock()
if err := s.reader.CommitRecords(ctx, crecords...); err != nil {
mu.Unlock()
return err
}
atomic.AddInt64(&cnt, -int64(len(crecords)))
crecords = crecords[:0]
mu.Unlock()
}
}
})
}()
for _, records := range mprecords {
precords := records
g.Go(func() error {
for _, record := range precords {
select {
case <-s.done:
return nil
case <-gctx.Done():
return gctx.Err()
default:
msg := pPool.Get().(*broker.Message)
msg.Header = nil
msg.Body = nil
p := &publication{topic: record.Topic, msg: msg}
if s.opts.BodyOnly {
p.msg.Body = record.Value
} else {
if err := s.kopts.Codec.Unmarshal(record.Value, p.msg); err != nil {
p.err = err
for _, fetch := range fetches {
for _, ftopic := range fetch.Topics {
for _, partition := range ftopic.Partitions {
precords := partition.Records
g.Go(func() error {
for _, record := range precords {
if atomic.LoadInt32(&done) == 1 {
return nil
}
p := pPool.Get().(*publication)
p.msg.Header = nil
p.msg.Body = nil
p.topic = s.topic
p.err = nil
p.ack = false
if s.opts.BodyOnly {
p.msg.Body = record.Value
} else {
if err := s.kopts.Codec.Unmarshal(record.Value, p.msg); err != nil {
p.err = err
p.msg.Body = record.Value
if eh != nil {
_ = eh(p)
if p.ack {
fillOffsets(offsets, record)
}
pPool.Put(p)
continue
} else {
if s.kopts.Logger.V(logger.ErrorLevel) {
s.kopts.Logger.Errorf(s.kopts.Context, "[kgo]: failed to unmarshal: %v", err)
}
}
pPool.Put(p)
return err
}
}
err = s.handler(p)
if err == nil && s.opts.AutoAck {
p.ack = true
} else if err != nil {
p.err = err
if eh != nil {
_ = eh(p)
if p.ack {
mu.Lock()
crecords = append(crecords, record)
mu.Unlock()
}
pPool.Put(msg)
return nil
} else {
if s.kopts.Logger.V(logger.ErrorLevel) {
s.kopts.Logger.Errorf(s.kopts.Context, "[kgo]: failed to unmarshal: %v", err)
s.kopts.Logger.Errorf(s.kopts.Context, "[kgo]: subscriber error: %v", err)
}
}
pPool.Put(msg)
return err
}
}
err = s.handler(p)
if err == nil && (s.opts.AutoAck || p.ack) {
mu.Lock()
crecords = append(crecords, record)
mu.Unlock()
}
if err != nil {
p.err = err
if eh != nil {
_ = eh(p)
} else {
if s.kopts.Logger.V(logger.ErrorLevel) {
s.kopts.Logger.Errorf(s.kopts.Context, "[kgo]: subscriber error: %v", err)
}
}
if p.ack {
mu.Lock()
crecords = append(crecords, record)
mu.Unlock()
fillOffsets(offsets, record)
}
pPool.Put(p)
}
pPool.Put(msg)
}
return nil
})
}
return nil
})
if err := g.Wait(); err != nil {
return err
}
}
}
if err := g.Wait(); err != nil {
return err
}
close(doneCh)
return s.reader.CommitRecords(ctx, crecords...)
return commitOffsets(s.reader, ctx, offsets)
}
func (k *kBroker) String() string {
@ -533,7 +557,6 @@ func NewBroker(opts ...broker.Option) broker.Broker {
kopts := []kgo.Opt{
kgo.BatchCompression(kgo.NoCompression()),
kgo.WithLogger(&mlogger{l: options.Logger, ctx: options.Context}),
kgo.RequiredAcks(kgo.AllISRAcks()),
kgo.RetryBackoffFn(
func() func(int) time.Duration {
var rng mrand.Rand

11
kgo_test.go
View File

@ -2,6 +2,7 @@ package kgo_test
import (
"context"
"fmt"
"os"
"strings"
"sync/atomic"
@ -56,7 +57,7 @@ func TestPubSub(t *testing.T) {
}
}()
/*
/*
msgs := make([]*broker.Message, 0, 600000)
for i := 0; i < 600000; i++ {
msgs = append(msgs, bm)
@ -66,7 +67,7 @@ func TestPubSub(t *testing.T) {
t.Fatal(err)
}
t.Skip()
*/
*/
done := make(chan bool, 1)
idx := int64(0)
fn := func(msg broker.Event) error {
@ -77,7 +78,7 @@ func TestPubSub(t *testing.T) {
sub, err := b.Subscribe(ctx, "test", fn,
broker.SubscribeAutoAck(true),
broker.SubscribeGroup("test23"),
broker.SubscribeGroup("test29"),
broker.SubscribeBodyOnly(true))
if err != nil {
t.Fatal(err)
@ -91,9 +92,13 @@ func TestPubSub(t *testing.T) {
ticker := time.NewTicker(2 * time.Minute)
defer ticker.Stop()
pticker := time.NewTicker(1 * time.Second)
defer pticker.Stop()
go func() {
for {
select {
case <-pticker.C:
fmt.Printf("processed %v\n", atomic.LoadInt64(&idx))
case <-ticker.C:
close(done)
}