unistack-org/micro
4
0
Fork 2
Code Issues 18 Pull Requests Actions Packages Projects Releases 324 Wiki Activity

Event stream updates (#1981)

Browse Source 
- auto and manual acking
- retry limits
This commit is contained in:
Dominic Wong
2020-09-02 13:28:54 +01:00
committed by GitHub
parent 84f52fd7ac
commit 6bdf33c4ee
8 changed files with 383 additions and 299 deletions
Download Patch File Download Diff File Expand all files Collapse all files

34
events/stream/nats/nats.go
View File

@@ -116,7 +116,8 @@ func (s *stream) Subscribe(topic string, opts ...events.SubscribeOption) (<-chan
// parse the options
options := events.SubscribeOptions{
Queue: uuid.New().String(),
Queue: uuid.New().String(),
AutoAck: true,
}
for _, o := range opts {
o(&options)
@@ -125,19 +126,43 @@ func (s *stream) Subscribe(topic string, opts ...events.SubscribeOption) (<-chan
// setup the subscriber
c := make(chan events.Event)
handleMsg := func(m *stan.Msg) {
// poison message handling
if options.GetRetryLimit() > -1 && m.Redelivered && int(m.RedeliveryCount) > options.GetRetryLimit() {
if logger.V(logger.ErrorLevel, logger.DefaultLogger) {
logger.Errorf("Message retry limit reached, discarding: %v", m.Sequence)
}
m.Ack() // ignoring error
return
}
// decode the message
var evt events.Event
if err := json.Unmarshal(m.Data, &evt); err != nil {
if logger.V(logger.ErrorLevel, logger.DefaultLogger) {
logger.Errorf("Error decoding message: %v", err)
}
// not ackknowledging the message is the way to indicate an error occured
// not acknowledging the message is the way to indicate an error occurred
return
}
if !options.AutoAck {
// set up the ack funcs
evt.SetAckFunc(func() error {
return m.Ack()
})
evt.SetNackFunc(func() error {
// noop. not acknowledging the message is the way to indicate an error occurred
// we have to wait for the ack wait to kick in before the message is resent
return nil
})
}
// push onto the channel and wait for the consumer to take the event off before we acknowledge it.
c <- evt
if !options.AutoAck {
return
}
if err := m.Ack(); err != nil && logger.V(logger.ErrorLevel, logger.DefaultLogger) {
logger.Errorf("Error acknowledging message: %v", err)
}
@@ -149,7 +174,10 @@ func (s *stream) Subscribe(topic string, opts ...events.SubscribeOption) (<-chan
stan.SetManualAckMode(),
}
if options.StartAtTime.Unix() > 0 {
stan.StartAtTime(options.StartAtTime)
subOpts = append(subOpts, stan.StartAtTime(options.StartAtTime))
}
if options.AckWait > 0 {
subOpts = append(subOpts, stan.AckWait(options.AckWait))
}
// connect the subscriber

151
events/stream/nats/nats_test.go
View File

@@ -1,150 +1,3 @@
// +build nats
package nats
import (
"net"
"os/exec"
"sync"
"testing"
"time"
"github.com/google/uuid"
"github.com/micro/go-micro/v3/events"
"github.com/stretchr/testify/assert"
)
type testPayload struct {
Message string
}
func TestStream(t *testing.T) {
_, err := exec.LookPath("nats-streaming-server")
if err != nil {
t.Skipf("Skipping nats test, nats-streaming-server binary is not detected")
}
conn, err := net.DialTimeout("tcp", ":4222", time.Millisecond*100)
if err != nil {
t.Skipf("Skipping nats test, could not connect to cluster on port 4222: %v", err)
}
if err := conn.Close(); err != nil {
t.Fatalf("Error closing test tcp connection to nats cluster")
}
stream, err := NewStream(ClusterID("test-cluster"))
assert.Nilf(t, err, "NewStream should not return an error")
assert.NotNilf(t, stream, "NewStream should return a stream object")
// TestMissingTopic will test the topic validation on publish
t.Run("TestMissingTopic", func(t *testing.T) {
err := stream.Publish("", nil)
assert.Equalf(t, err, events.ErrMissingTopic, "Publishing to a blank topic should return an error")
})
// TestSubscribeTopic will publish a message to the test topic. The subscriber will subscribe to the
// same test topic.
t.Run("TestSubscribeTopic", func(t *testing.T) {
payload := &testPayload{Message: "HelloWorld"}
metadata := map[string]string{"foo": "bar"}
// create the subscriber
evChan, err := stream.Subscribe("test")
assert.Nilf(t, err, "Subscribe should not return an error")
// setup the subscriber async
var wg sync.WaitGroup
go func() {
timeout := time.NewTimer(time.Millisecond * 250)
select {
case event, _ := <-evChan:
assert.NotNilf(t, event, "The message was nil")
assert.Equal(t, event.Metadata, metadata, "Metadata didn't match")
var result testPayload
err = event.Unmarshal(&result)
assert.Nil(t, err, "Error decoding result")
assert.Equal(t, result, *payload, "Payload didn't match")
wg.Done()
case <-timeout.C:
t.Fatalf("Event was not recieved")
}
}()
err = stream.Publish("test", payload, events.WithMetadata(metadata))
assert.Nil(t, err, "Publishing a valid message should not return an error")
wg.Add(1)
// wait for the subscriber to recieve the message or timeout
wg.Wait()
})
// TestSubscribeQueue will publish a message to a random topic. Two subscribers will then consume
// the message from the firehose topic with different queues. The second subscriber will be registered
// after the message is published to test durability.
t.Run("TestSubscribeQueue", func(t *testing.T) {
topic := uuid.New().String()
payload := &testPayload{Message: "HelloWorld"}
metadata := map[string]string{"foo": "bar"}
// create the first subscriber
evChan1, err := stream.Subscribe(topic)
assert.Nilf(t, err, "Subscribe should not return an error")
// setup the subscriber async
var wg sync.WaitGroup
go func() {
timeout := time.NewTimer(time.Millisecond * 250)
select {
case event, _ := <-evChan1:
assert.NotNilf(t, event, "The message was nil")
assert.Equal(t, event.Metadata, metadata, "Metadata didn't match")
var result testPayload
err = event.Unmarshal(&result)
assert.Nil(t, err, "Error decoding result")
assert.Equal(t, result, *payload, "Payload didn't match")
wg.Done()
case <-timeout.C:
t.Fatalf("Event was not recieved")
}
}()
err = stream.Publish(topic, payload, events.WithMetadata(metadata))
assert.Nil(t, err, "Publishing a valid message should not return an error")
wg.Add(2)
// create the second subscriber
evChan2, err := stream.Subscribe(topic,
events.WithQueue("second_queue"),
events.WithStartAtTime(time.Now().Add(time.Minute*-1)),
)
assert.Nilf(t, err, "Subscribe should not return an error")
go func() {
timeout := time.NewTimer(time.Millisecond * 250)
select {
case event, _ := <-evChan2:
assert.NotNilf(t, event, "The message was nil")
assert.Equal(t, event.Metadata, metadata, "Metadata didn't match")
var result testPayload
err = event.Unmarshal(&result)
assert.Nil(t, err, "Error decoding result")
assert.Equal(t, result, *payload, "Payload didn't match")
wg.Done()
case <-timeout.C:
t.Fatalf("Event was not recieved")
}
}()
// wait for the subscriber to recieve the message or timeout
wg.Wait()
})
}