cpu and memory optimizations

Signed-off-by: Vasiliy Tolstov <v.tolstov@unistack.org>

go.mod

@@ -4,9 +4,10 @@ go 1.16
 
 require (
 	github.com/twmb/franz-go v0.10.3-0.20210825060253-d5e80b38ca2b
-	github.com/twmb/franz-go/pkg/kmsg v0.0.0-20210825163214-e185676761dd // indirect
+	github.com/twmb/franz-go/pkg/kmsg v0.0.0-20210826221812-c6df11da978a
 	github.com/unistack-org/micro-codec-json/v3 v3.2.5
 	github.com/unistack-org/micro-proto v0.0.8 // indirect
 	github.com/unistack-org/micro/v3 v3.6.3
 	golang.org/x/sync v0.0.0-20210220032951-036812b2e83c
+	google.golang.org/protobuf v1.27.1 // indirect
 )

go.sum

@@ -61,6 +61,8 @@ github.com/twmb/franz-go v0.10.3-0.20210825060253-d5e80b38ca2b/go.mod h1:Txc5/v0
 github.com/twmb/franz-go/pkg/kmsg v0.0.0-20210823212011-0d01f7456b4d/go.mod h1:SxG/xJKhgPu25SamAq0rrucfp7lbzCpEXOC+vH/ELrY=
 github.com/twmb/franz-go/pkg/kmsg v0.0.0-20210825163214-e185676761dd h1:8TvJbg0dnOaq4Oex1wnYejPokyUQ4zG2OQUfD76yq4M=
 github.com/twmb/franz-go/pkg/kmsg v0.0.0-20210825163214-e185676761dd/go.mod h1:SxG/xJKhgPu25SamAq0rrucfp7lbzCpEXOC+vH/ELrY=
+github.com/twmb/franz-go/pkg/kmsg v0.0.0-20210826221812-c6df11da978a h1:UBO58MRI/GH5eD5+fKXZoJMGyEcsUs9LQnHJbNL0yKs=
+github.com/twmb/franz-go/pkg/kmsg v0.0.0-20210826221812-c6df11da978a/go.mod h1:SxG/xJKhgPu25SamAq0rrucfp7lbzCpEXOC+vH/ELrY=
 github.com/twmb/go-rbtree v1.0.0 h1:KxN7dXJ8XaZ4cvmHV1qqXTshxX3EBvX/toG5+UR49Mg=
 github.com/twmb/go-rbtree v1.0.0/go.mod h1:UlIAI8gu3KRPkXSobZnmJfVwCJgEhD/liWzT5ppzIyc=
 github.com/unistack-org/micro-codec-json/v3 v3.2.5 h1:WOilhbL0YSu58iIQIIxpawRYZyx6CR16tCpbX4ai3Vc=
@@ -90,6 +92,7 @@ golang.org/x/net v0.0.0-20210813160813-60bc85c4be6d/go.mod h1:9nx3DQGgdP8bBQD5qx
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
 golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
+golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c h1:5KslGYwFpkhGh+Q16bwMP3cOontH8FOep7tGV86Y7SQ=
 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
@@ -130,6 +133,8 @@ google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlba
 google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw=
 google.golang.org/protobuf v1.26.0 h1:bxAC2xTBsZGibn2RTntX0oH50xLsqy1OxA9tTL3p/lk=
 google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
+google.golang.org/protobuf v1.27.1 h1:SnqbnDw1V7RiZcXPx5MEeqPv2s79L9i7BJUlG/+RurQ=
+google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
 gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=

kgo.go

@@ -10,7 +10,9 @@ import (
 	"sync/atomic"
 	"time"
 
+	kerr "github.com/twmb/franz-go/pkg/kerr"
 	kgo "github.com/twmb/franz-go/pkg/kgo"
+	kmsg "github.com/twmb/franz-go/pkg/kmsg"
 	"github.com/unistack-org/micro/v3/broker"
 	"github.com/unistack-org/micro/v3/logger"
 	"github.com/unistack-org/micro/v3/metadata"
@@ -21,7 +23,7 @@ import (
 
 var pPool = sync.Pool{
 	New: func() interface{} {
-		return &broker.Message{}
+		return &publication{msg: &broker.Message{}}
 	},
 }
 
@@ -336,7 +338,7 @@ func (k *kBroker) Subscribe(ctx context.Context, topic string, handler broker.Ha
 		kgo.ConsumeResetOffset(kgo.NewOffset().AtStart()),
 		kgo.DisableAutoCommit(),
 		kgo.FetchMaxWait(1*time.Second),
-		kgo.KeepControlRecords(),
+		// kgo.KeepControlRecords(),
 		kgo.Balancers(kgo.CooperativeStickyBalancer(), kgo.StickyBalancer()),
 		kgo.FetchIsolationLevel(kgo.ReadUncommitted()),
 	//	kgo.WithHooks(&metrics{meter: k.opts.Meter}),
@@ -348,7 +350,7 @@ func (k *kBroker) Subscribe(ctx context.Context, topic string, handler broker.Ha
 		return nil, err
 	}
 
-	sub := &subscriber{done: make(chan struct{}), opts: options, reader: reader, handler: handler, kopts: k.opts}
+	sub := &subscriber{topic: topic, done: make(chan struct{}), opts: options, reader: reader, handler: handler, kopts: k.opts}
 	go sub.run(ctx)
 
 	k.Lock()
@@ -390,21 +392,6 @@ func (s *subscriber) run(ctx context.Context) {
 func (s *subscriber) handleFetches(ctx context.Context, fetches kgo.Fetches) error {
 	var err error
 
-	mprecords := make(map[int32][]*kgo.Record)
-
-	cnt := int64(0)
-	for _, fetch := range fetches {
-		for _, ftopic := range fetch.Topics {
-			for _, partition := range ftopic.Partitions {
-				mprecords[partition.Partition] = append(mprecords[partition.Partition], partition.Records...)
-				cnt += int64(len(partition.Records))
-			}
-		}
-	}
-
-	// preallocate optimistic
-	crecords := make([]*kgo.Record, 0, 10000)
-
 	eh := s.kopts.ErrorHandler
 	if s.opts.ErrorHandler != nil {
 		eh = s.opts.ErrorHandler
@@ -412,6 +399,8 @@ func (s *subscriber) handleFetches(ctx context.Context, fetches kgo.Fetches) err
 
 	var mu sync.Mutex
 
+	done := int32(0)
+	doneCh := make(chan struct{})
 	g, gctx := errgroup.WithContext(ctx)
 
 	td := DefaultCommitInterval
@@ -421,107 +410,142 @@ func (s *subscriber) handleFetches(ctx context.Context, fetches kgo.Fetches) err
 		}
 	}
 
+	// ticker for commit offsets
 	ticker := time.NewTicker(td)
 	defer ticker.Stop()
 
-	g.Go(func() error {
+	offsets := make(map[string]map[int32]kgo.EpochOffset)
+	offsets[s.topic] = make(map[int32]kgo.EpochOffset)
+
+	fillOffsets := func(off map[string]map[int32]kgo.EpochOffset, rec *kgo.Record) {
+		mu.Lock()
+		if at, ok := off[s.topic][rec.Partition]; ok {
+			if at.Epoch > rec.LeaderEpoch || at.Epoch == rec.LeaderEpoch && at.Offset > rec.Offset {
+				mu.Unlock()
+				return
+			}
+		}
+		off[s.topic][rec.Partition] = kgo.EpochOffset{Epoch: rec.LeaderEpoch, Offset: rec.Offset + 1}
+		mu.Unlock()
+	}
+
+	commitOffsets := func(cl *kgo.Client, ctx context.Context, off map[string]map[int32]kgo.EpochOffset) error {
+		var rerr error
+
+		mu.Lock()
+		offsets := off
+		mu.Unlock()
+
+		cl.CommitOffsetsSync(ctx, offsets, func(_ *kgo.Client, _ *kmsg.OffsetCommitRequest, resp *kmsg.OffsetCommitResponse, err error) {
+			if err != nil {
+				rerr = err
+				return
+			}
+
+			for _, topic := range resp.Topics {
+				for _, partition := range topic.Partitions {
+					if err := kerr.ErrorForCode(partition.ErrorCode); err != nil {
+						rerr = err
+						return
+					}
+				}
+			}
+		})
+
+		return rerr
+	}
+
+	go func() {
 		for {
 			select {
 			case <-gctx.Done():
-				return gctx.Err()
+				return
 			case <-s.done:
-				mu.Lock()
+				atomic.StoreInt32(&done, 1)
-				err := s.reader.CommitRecords(ctx, crecords...)
+				if err := commitOffsets(s.reader, ctx, offsets); err != nil && s.kopts.Logger.V(logger.ErrorLevel) {
-				mu.Unlock()
+					s.kopts.Logger.Errorf(s.kopts.Context, "[kgo]: failed to commit offsets: %v", err)
-				return err
+				}
+				return
+			case <-doneCh:
+				return
 			case <-ticker.C:
-				v := atomic.LoadInt64(&cnt)
+				if err := commitOffsets(s.reader, ctx, offsets); err != nil {
-				if v == 0 {
+					if s.kopts.Logger.V(logger.ErrorLevel) {
-					return nil
+						s.kopts.Logger.Errorf(s.kopts.Context, "[kgo]: failed to commit offsets: %v", err)
+					}
+					return
 				}
-				mu.Lock()
-				if err := s.reader.CommitRecords(ctx, crecords...); err != nil {
-					mu.Unlock()
-					return err
-				}
-				atomic.AddInt64(&cnt, -int64(len(crecords)))
-				crecords = crecords[:0]
-				mu.Unlock()
 			}
 		}
-	})
+	}()
 
-	for _, records := range mprecords {
+	for _, fetch := range fetches {
-		precords := records
+		for _, ftopic := range fetch.Topics {
-		g.Go(func() error {
+			for _, partition := range ftopic.Partitions {
-			for _, record := range precords {
+				precords := partition.Records
-				select {
+				g.Go(func() error {
-				case <-s.done:
+					for _, record := range precords {
-					return nil
+						if atomic.LoadInt32(&done) == 1 {
-				case <-gctx.Done():
+							return nil
-					return gctx.Err()
+						}
-				default:
+						p := pPool.Get().(*publication)
-					msg := pPool.Get().(*broker.Message)
+						p.msg.Header = nil
-					msg.Header = nil
+						p.msg.Body = nil
-					msg.Body = nil
+						p.topic = s.topic
-					p := &publication{topic: record.Topic, msg: msg}
+						p.err = nil
-					if s.opts.BodyOnly {
+						p.ack = false
-						p.msg.Body = record.Value
+						if s.opts.BodyOnly {
-					} else {
-						if err := s.kopts.Codec.Unmarshal(record.Value, p.msg); err != nil {
-							p.err = err
 							p.msg.Body = record.Value
+						} else {
+							if err := s.kopts.Codec.Unmarshal(record.Value, p.msg); err != nil {
+								p.err = err
+								p.msg.Body = record.Value
+								if eh != nil {
+									_ = eh(p)
+									if p.ack {
+										fillOffsets(offsets, record)
+									}
+									pPool.Put(p)
+									continue
+								} else {
+									if s.kopts.Logger.V(logger.ErrorLevel) {
+										s.kopts.Logger.Errorf(s.kopts.Context, "[kgo]: failed to unmarshal: %v", err)
+									}
+								}
+								pPool.Put(p)
+								return err
+							}
+						}
+						err = s.handler(p)
+						if err == nil && s.opts.AutoAck {
+							p.ack = true
+						} else if err != nil {
+							p.err = err
 							if eh != nil {
 								_ = eh(p)
-								if p.ack {
-									mu.Lock()
-									crecords = append(crecords, record)
-									mu.Unlock()
-								}
-								pPool.Put(msg)
-								return nil
 							} else {
 								if s.kopts.Logger.V(logger.ErrorLevel) {
-									s.kopts.Logger.Errorf(s.kopts.Context, "[kgo]: failed to unmarshal: %v", err)
+									s.kopts.Logger.Errorf(s.kopts.Context, "[kgo]: subscriber error: %v", err)
 								}
 							}
-							pPool.Put(msg)
-							return err
-						}
-					}
-					err = s.handler(p)
-					if err == nil && (s.opts.AutoAck || p.ack) {
-						mu.Lock()
-						crecords = append(crecords, record)
-						mu.Unlock()
-					}
-					if err != nil {
-						p.err = err
-						if eh != nil {
-							_ = eh(p)
-						} else {
-							if s.kopts.Logger.V(logger.ErrorLevel) {
-								s.kopts.Logger.Errorf(s.kopts.Context, "[kgo]: subscriber error: %v", err)
-							}
 						}
 						if p.ack {
-							mu.Lock()
+							fillOffsets(offsets, record)
-							crecords = append(crecords, record)
-							mu.Unlock()
 						}
+						pPool.Put(p)
 					}
-					pPool.Put(msg)
+					return nil
-				}
+				})
 			}
-			return nil
+			if err := g.Wait(); err != nil {
-		})
+				return err
+			}
+		}
 	}
 
-	if err := g.Wait(); err != nil {
+	close(doneCh)
-		return err
-	}
 
-	return s.reader.CommitRecords(ctx, crecords...)
+	return commitOffsets(s.reader, ctx, offsets)
 }
 
 func (k *kBroker) String() string {
@@ -533,7 +557,6 @@ func NewBroker(opts ...broker.Option) broker.Broker {
 	kopts := []kgo.Opt{
 		kgo.BatchCompression(kgo.NoCompression()),
 		kgo.WithLogger(&mlogger{l: options.Logger, ctx: options.Context}),
-		kgo.RequiredAcks(kgo.AllISRAcks()),
 		kgo.RetryBackoffFn(
 			func() func(int) time.Duration {
 				var rng mrand.Rand

kgo_test.go

@@ -2,6 +2,7 @@ package kgo_test
 
 import (
 	"context"
+	"fmt"
 	"os"
 	"strings"
 	"sync/atomic"
@@ -56,7 +57,7 @@ func TestPubSub(t *testing.T) {
 		}
 	}()
 
-/*
+	/*
 			msgs := make([]*broker.Message, 0, 600000)
 			for i := 0; i < 600000; i++ {
 				msgs = append(msgs, bm)
@@ -66,7 +67,7 @@ func TestPubSub(t *testing.T) {
 				t.Fatal(err)
 			}
 		t.Skip()
-*/
+	*/
 	done := make(chan bool, 1)
 	idx := int64(0)
 	fn := func(msg broker.Event) error {
@@ -77,7 +78,7 @@ func TestPubSub(t *testing.T) {
 
 	sub, err := b.Subscribe(ctx, "test", fn,
 		broker.SubscribeAutoAck(true),
-		broker.SubscribeGroup("test23"),
+		broker.SubscribeGroup("test29"),
 		broker.SubscribeBodyOnly(true))
 	if err != nil {
 		t.Fatal(err)
@@ -91,9 +92,13 @@ func TestPubSub(t *testing.T) {
 	ticker := time.NewTicker(2 * time.Minute)
 	defer ticker.Stop()
 
+	pticker := time.NewTicker(1 * time.Second)
+	defer pticker.Stop()
 	go func() {
 		for {
 			select {
+			case <-pticker.C:
+				fmt.Printf("processed %v\n", atomic.LoadInt64(&idx))
 			case <-ticker.C:
 				close(done)
 			}