broker/segmentio: tmp revert

Signed-off-by: Vasiliy Tolstov <v.tolstov@unistack.org>

segmentio.go

@@ -4,6 +4,7 @@ package segmentio
 import (
 	"context"
 	"errors"
+	"io"
 	"sync"
 
 	"github.com/google/uuid"
@@ -28,9 +29,15 @@ type kBroker struct {
 }
 
 type subscriber struct {
+	k         *kBroker
 	group     *kafka.ConsumerGroup
 	t         string
 	opts      broker.SubscribeOptions
+	offset    int64
+	gen       *kafka.Generation
+	partition int
+	handler   broker.Handler
+	reader    *kafka.Reader
 }
 
 type publication struct {
@@ -39,6 +46,8 @@ type publication struct {
 	m      *broker.Message
 	ctx    context.Context
 	gen    *kafka.Generation
+	reader *kafka.Reader
+	km     kafka.Message
 	mp     map[string]map[int]int64 // for commit offsets
 }
 
@@ -55,7 +64,8 @@ func (p *publication) Message() *broker.Message {
 }
 
 func (p *publication) Ack() error {
-	return p.gen.CommitOffsets(p.mp)
+	//return p.gen.CommitOffsets(p.mp)
+	return p.reader.CommitMessages(p.ctx, p.km)
 }
 
 func (p *publication) Error() error {
@@ -71,7 +81,8 @@ func (s *subscriber) Topic() string {
 }
 
 func (s *subscriber) Unsubscribe() error {
-	return s.group.Close()
+	//return s.group.Close()
+	return s.reader.Close()
 }
 
 func (k *kBroker) Address() string {
@@ -191,63 +202,32 @@ func (k *kBroker) Subscribe(topic string, handler broker.Handler, opts ...broker
 		o(&opt)
 	}
 
-	gcfg := kafka.ConsumerGroupConfig{
+	cfg := k.readerConfig
-		ID:                    opt.Queue,
+	cfg.Topic = topic
-		WatchPartitionChanges: true,
+	cfg.GroupID = opt.Queue
-		Brokers:               k.readerConfig.Brokers,
+	cfg.WatchPartitionChanges = true
-		Topics:                []string{topic},
+	cfg.MaxAttempts = 1
-	}
+	if err := cfg.Validate(); err != nil {
-	if err := gcfg.Validate(); err != nil {
 		return nil, err
 	}
 
-	group, err := kafka.NewConsumerGroup(gcfg)
+	reader := kafka.NewReader(cfg)
-	if err != nil {
+	sub := &subscriber{opts: opt, t: topic, reader: reader}
-		return nil, err
-	}
-
-	sub := &subscriber{group: group, opts: opt, t: topic}
-
-	chErr := make(chan error)
 
 	go func() {
 		for {
-			gen, err := group.Next(k.opts.Context)
+			select {
-			if err == kafka.ErrGroupClosed {
+			case <-k.opts.Context.Done():
-				chErr <- nil
+				return
+			default:
+				msg, err := reader.FetchMessage(k.opts.Context)
+				if err != nil && err == io.EOF {
 					return
 				} else if err != nil {
-				chErr <- err
+					logger.Errorf("[kafka] subscribe error: %v", err)
-				return
 				}
-			chErr <- nil
+				p := &publication{t: topic, ctx: k.opts.Context, reader: reader, km: msg}
 
-			assignments := gen.Assignments[topic]
-			for _, assignment := range assignments {
-				partition, offset := assignment.ID, assignment.Offset
-				p := &publication{t: topic, ctx: k.opts.Context, gen: gen}
-				p.mp = map[string]map[int]int64{p.t: {partition: offset}}
-
-				gen.Start(func(ctx context.Context) {
-					// create reader for this partition.
-					cfg := k.readerConfig
-					cfg.Topic = topic
-					cfg.Partition = partition
-					cfg.GroupID = ""
-					reader := kafka.NewReader(cfg)
-					defer reader.Close()
-					// seek to the last committed offset for this partition.
-					reader.SetOffset(offset)
-					for {
-						msg, err := reader.ReadMessage(ctx)
-						switch err {
-						case kafka.ErrGenerationEnded:
-							// generation has ended
-							if logger.V(logger.DebugLevel, logger.DefaultLogger) {
-								logger.Debug("[kafka] subscription closed")
-							}
-							return
-						case nil:
 				var m broker.Message
 				eh := k.opts.ErrorHandler
 				p.m = &m
@@ -278,21 +258,158 @@ func (k *kBroker) Subscribe(topic string, handler broker.Handler, opts ...broker
 						}
 					}
 				}
-						}
+
-					}
-				})
 			}
 		}
 	}()
 
-	err = <-chErr
+	return sub, nil
+}
+
+/*
+func (k *kBroker) Subscribe(topic string, handler broker.Handler, opts ...broker.SubscribeOption) (broker.Subscriber, error) {
+	opt := broker.SubscribeOptions{
+		AutoAck: true,
+		Queue:   uuid.New().String(),
+	}
+	for _, o := range opts {
+		o(&opt)
+	}
+
+	gcfg := kafka.ConsumerGroupConfig{
+		ID:                    opt.Queue,
+		WatchPartitionChanges: true,
+		Brokers:               k.readerConfig.Brokers,
+		Topics:                []string{topic},
+		GroupBalancers:        []kafka.GroupBalancer{kafka.RangeGroupBalancer{}},
+	}
+	if err := gcfg.Validate(); err != nil {
+		return nil, err
+	}
+
+	sub := &subscriber{k: k, opts: opt, t: topic, handler: handler}
+
+	chErr := make(chan error)
+
+	go func() {
+		for {
+			logger.Info("new consumer group")
+			group, err := kafka.NewConsumerGroup(gcfg)
+			if err != nil {
+				chErr <- err
+				time.Sleep(1 * time.Second)
+				continue
+			}
+			logger.Info("group next")
+			gen, err := group.Next(k.opts.Context)
+			if err == kafka.ErrGroupClosed {
+				chErr <- nil
+				time.Sleep(1 * time.Second)
+				continue
+			} else if err != nil {
+				chErr <- err
+				time.Sleep(1 * time.Second)
+				continue
+			}
+			chErr <- nil
+			logger.Info("gen assign")
+			assignments := gen.Assignments[topic]
+			for _, assignment := range assignments {
+				partition, offset := assignment.ID, assignment.Offset
+				sub.offset = offset
+				sub.partition = partition
+				sub.gen = gen
+				logger.Infof("gen start part %v off %v", partition, offset)
+				gen.Start(sub.run)
+			}
+		}
+	}()
+
+	err := <-chErr
 	if err != nil {
 		return nil, err
 	}
 
+	go func() {
+		for {
+			select {
+			case <-k.opts.Context.Done():
+				return
+			case err := <-chErr:
+				if err != nil {
+					logger.Error(err)
+				}
+			}
+		}
+	}()
+
 	return sub, nil
 }
 
+func (s *subscriber) run(ctx context.Context) {
+	// create reader for this partition.
+	cfg := s.k.readerConfig
+	cfg.Topic = s.t
+	cfg.Partition = s.partition
+	cfg.GroupID = ""
+	p := &publication{t: s.t, ctx: s.k.opts.Context, gen: s.gen}
+	p.mp = map[string]map[int]int64{p.t: {s.partition: s.offset}}
+
+	reader := kafka.NewReader(cfg)
+	defer reader.Close()
+	logger.Info("set offset")
+	// seek to the last committed offset for this partition.
+	reader.SetOffset(s.offset)
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		default:
+			msg, err := reader.ReadMessage(ctx)
+			switch err {
+			case kafka.ErrGenerationEnded:
+				logger.Infof("reader err: %v", err)
+				// generation has ended
+				if logger.V(logger.DebugLevel, logger.DefaultLogger) {
+					logger.Debug("[kafka] subscription closed")
+				}
+				return
+			case nil:
+				var m broker.Message
+				eh := s.k.opts.ErrorHandler
+				p.m = &m
+				if err := s.k.opts.Codec.Unmarshal(msg.Value, &m); err != nil {
+					p.err = err
+					p.m.Body = msg.Value
+					if eh != nil {
+						eh(p)
+					} else {
+						if logger.V(logger.ErrorLevel, logger.DefaultLogger) {
+							logger.Errorf("[kafka]: failed to unmarshal: %v", err)
+						}
+					}
+					continue
+				}
+				err = s.handler(p)
+				if err == nil && s.opts.AutoAck {
+					if err = p.Ack(); err != nil {
+						logger.Errorf("[kafka]: unable to commit msg: %v", err)
+					}
+				} else if err != nil {
+					p.err = err
+					if eh != nil {
+						eh(p)
+					} else {
+						if logger.V(logger.ErrorLevel, logger.DefaultLogger) {
+							logger.Errorf("[kafka]: subscriber error: %v", err)
+						}
+					}
+				}
+			}
+		}
+	}
+}
+*/
 func (k *kBroker) String() string {
 	return "kafka"
 }