unistack-org/micro-tests
4
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

update all

Browse Source 
Signed-off-by: Vasiliy Tolstov <v.tolstov@unistack.org>
This commit is contained in:
Василий Толстов
2021-07-31 15:17:18 +03:00
parent f1db855826
commit 03fe6da356
8 changed files with 230 additions and 475 deletions
Download Patch File Download Diff File Expand all files Collapse all files

389
broker/http/http_test.go
View File

@@ -1,389 +0,0 @@
package http_test
import (
"context"
"sync"
"testing"
"time"
"github.com/google/uuid"
http "github.com/unistack-org/micro-broker-http/v3"
jsoncodec "github.com/unistack-org/micro-codec-json/v3"
"github.com/unistack-org/micro/v3/broker"
"github.com/unistack-org/micro/v3/register"
)
var (
// mock data
testData = map[string][]*register.Service{
"foo": {
{
Name: "foo",
Version: "1.0.0",
Nodes: []*register.Node{
{
ID: "foo-1.0.0-123",
Address: "localhost:9999",
},
{
ID: "foo-1.0.0-321",
Address: "localhost:9999",
},
},
},
{
Name: "foo",
Version: "1.0.1",
Nodes: []*register.Node{
{
ID: "foo-1.0.1-321",
Address: "localhost:6666",
},
},
},
{
Name: "foo",
Version: "1.0.3",
Nodes: []*register.Node{
{
ID: "foo-1.0.3-345",
Address: "localhost:8888",
},
},
},
},
}
)
func newTestRegister() register.Register {
return register.NewRegister()
}
func sub(be *testing.B, c int) {
be.StopTimer()
m := newTestRegister()
b := http.NewBroker(broker.Codec(jsoncodec.NewCodec()), broker.Register(m))
topic := uuid.New().String()
if err := b.Init(); err != nil {
be.Fatalf("Unexpected init error: %v", err)
}
if err := b.Connect(context.TODO()); err != nil {
be.Fatalf("Unexpected connect error: %v", err)
}
msg := &broker.Message{
Header: map[string]string{
"Content-Type": "application/json",
},
Body: []byte(`{"message": "Hello World"}`),
}
var subs []broker.Subscriber
done := make(chan bool, c)
for i := 0; i < c; i++ {
sub, err := b.Subscribe(context.TODO(), topic, func(p broker.Event) error {
done <- true
m := p.Message()
if string(m.Body) != string(msg.Body) {
be.Fatalf("Unexpected msg %s, expected %s", string(m.Body), string(msg.Body))
}
return nil
}, broker.SubscribeGroup("shared"))
if err != nil {
be.Fatalf("Unexpected subscribe error: %v", err)
}
subs = append(subs, sub)
}
for i := 0; i < be.N; i++ {
be.StartTimer()
if err := b.Publish(context.TODO(), topic, msg); err != nil {
be.Fatalf("Unexpected publish error: %v", err)
}
<-done
be.StopTimer()
}
for _, sub := range subs {
sub.Unsubscribe(context.TODO())
}
if err := b.Disconnect(context.TODO()); err != nil {
be.Fatalf("Unexpected disconnect error: %v", err)
}
}
func pub(be *testing.B, c int) {
be.StopTimer()
m := newTestRegister()
b := http.NewBroker(broker.Codec(jsoncodec.NewCodec()), broker.Register(m))
topic := uuid.New().String()
if err := b.Init(); err != nil {
be.Fatalf("Unexpected init error: %v", err)
}
if err := b.Connect(context.TODO()); err != nil {
be.Fatalf("Unexpected connect error: %v", err)
}
msg := &broker.Message{
Header: map[string]string{
"Content-Type": "application/json",
},
Body: []byte(`{"message": "Hello World"}`),
}
done := make(chan bool, c*4)
sub, err := b.Subscribe(context.TODO(), topic, func(p broker.Event) error {
done <- true
m := p.Message()
if string(m.Body) != string(msg.Body) {
be.Fatalf("Unexpected msg %s, expected %s", string(m.Body), string(msg.Body))
}
return nil
}, broker.SubscribeGroup("shared"))
if err != nil {
be.Fatalf("Unexpected subscribe error: %v", err)
}
var wg sync.WaitGroup
ch := make(chan int, c*4)
be.StartTimer()
for i := 0; i < c; i++ {
go func() {
for range ch {
if err := b.Publish(context.TODO(), topic, msg); err != nil {
be.Fatalf("Unexpected publish error: %v", err)
}
select {
case <-done:
case <-time.After(time.Second):
}
wg.Done()
}
}()
}
for i := 0; i < be.N; i++ {
wg.Add(1)
ch <- i
}
wg.Wait()
be.StopTimer()
sub.Unsubscribe(context.TODO())
close(ch)
close(done)
if err := b.Disconnect(context.TODO()); err != nil {
be.Fatalf("Unexpected disconnect error: %v", err)
}
}
func TestBroker(t *testing.T) {
m := newTestRegister()
b := http.NewBroker(broker.Codec(jsoncodec.NewCodec()), broker.Register(m))
if err := b.Init(); err != nil {
t.Fatalf("Unexpected init error: %v", err)
}
if err := b.Connect(context.TODO()); err != nil {
t.Fatalf("Unexpected connect error: %v", err)
}
msg := &broker.Message{
Header: map[string]string{
"Content-Type": "application/json",
},
Body: []byte(`{"message":"Hello World"}`),
}
done := make(chan struct{})
sub, err := b.Subscribe(context.TODO(), "test", func(p broker.Event) error {
m := p.Message()
if string(m.Body) != string(msg.Body) {
t.Errorf("Unexpected msg %s, expected %s", string(m.Body), string(msg.Body))
}
close(done)
return nil
})
if err != nil {
t.Fatalf("Unexpected subscribe error: %v", err)
}
if err := b.Publish(context.TODO(), "test", msg); err != nil {
t.Fatalf("Unexpected publish error: %v", err)
}
<-done
if err != nil {
t.Error(err)
}
sub.Unsubscribe(context.TODO())
if err := b.Disconnect(context.TODO()); err != nil {
t.Fatalf("Unexpected disconnect error: %v", err)
}
}
func TestConcurrentSubBroker(t *testing.T) {
m := newTestRegister()
b := http.NewBroker(broker.Codec(jsoncodec.NewCodec()), broker.Register(m))
if err := b.Init(); err != nil {
t.Fatalf("Unexpected init error: %v", err)
}
if err := b.Connect(context.TODO()); err != nil {
t.Fatalf("Unexpected connect error: %v", err)
}
msg := &broker.Message{
Header: map[string]string{
"Content-Type": "application/json",
},
Body: []byte(`{"message":"Hello World"}`),
}
var subs []broker.Subscriber
var wg sync.WaitGroup
for i := 0; i < 10; i++ {
sub, err := b.Subscribe(context.TODO(), "test", func(p broker.Event) error {
defer wg.Done()
m := p.Message()
if string(m.Body) != string(msg.Body) {
t.Errorf("Unexpected msg %s, expected %s", string(m.Body), string(msg.Body))
}
return nil
})
if err != nil {
t.Fatalf("Unexpected subscribe error: %v", err)
}
wg.Add(1)
subs = append(subs, sub)
}
if err := b.Publish(context.TODO(), "test", msg); err != nil {
t.Fatalf("Unexpected publish error: %v", err)
}
wg.Wait()
for _, sub := range subs {
sub.Unsubscribe(context.TODO())
}
if err := b.Disconnect(context.TODO()); err != nil {
t.Fatalf("Unexpected disconnect error: %v", err)
}
}
func TestConcurrentPubBroker(t *testing.T) {
m := newTestRegister()
b := http.NewBroker(broker.Codec(jsoncodec.NewCodec()), broker.Register(m))
if err := b.Init(); err != nil {
t.Fatalf("Unexpected init error: %v", err)
}
if err := b.Connect(context.TODO()); err != nil {
t.Fatalf("Unexpected connect error: %v", err)
}
msg := &broker.Message{
Header: map[string]string{
"Content-Type": "application/json",
},
Body: []byte(`{"message":"Hello World"}`),
}
var wg sync.WaitGroup
sub, err := b.Subscribe(context.TODO(), "test", func(p broker.Event) error {
defer wg.Done()
m := p.Message()
if string(m.Body) != string(msg.Body) {
t.Fatalf("Unexpected msg %s, expected %s", string(m.Body), string(msg.Body))
}
return nil
})
if err != nil {
t.Fatalf("Unexpected subscribe error: %v", err)
}
for i := 0; i < 10; i++ {
wg.Add(1)
if err := b.Publish(context.TODO(), "test", msg); err != nil {
t.Fatalf("Unexpected publish error: %v", err)
}
}
wg.Wait()
sub.Unsubscribe(context.TODO())
if err := b.Disconnect(context.TODO()); err != nil {
t.Fatalf("Unexpected disconnect error: %v", err)
}
}
func BenchmarkSub1(b *testing.B) {
sub(b, 1)
}
func BenchmarkSub8(b *testing.B) {
sub(b, 8)
}
func BenchmarkSub32(b *testing.B) {
sub(b, 32)
}
func BenchmarkSub64(b *testing.B) {
sub(b, 64)
}
func BenchmarkSub128(b *testing.B) {
sub(b, 128)
}
func BenchmarkPub1(b *testing.B) {
pub(b, 1)
}
func BenchmarkPub8(b *testing.B) {
pub(b, 8)
}
func BenchmarkPub32(b *testing.B) {
pub(b, 32)
}
func BenchmarkPub64(b *testing.B) {
pub(b, 64)
}
func BenchmarkPub128(b *testing.B) {
pub(b, 128)
}

161
broker/segmentio/broker_test.go
View File

@@ -4,6 +4,7 @@ import (
"context"
"fmt"
"io"
"log"
"net/http"
"os"
"strings"
@@ -36,6 +37,157 @@ var (
}
)
func TestConsumerGroup(t *testing.T) {
topic := fmt.Sprintf("test_topic")
if tr := os.Getenv("INTEGRATION_TESTS"); len(tr) > 0 {
t.Skip()
}
logger.DefaultLogger.Init(logger.WithLevel(logger.TraceLevel))
ctx := context.Background()
var addrs []string
if addr := os.Getenv("BROKER_ADDRS"); len(addr) == 0 {
addrs = []string{"127.0.0.1:9092"}
} else {
addrs = strings.Split(addr, ",")
}
meter.DefaultMeter = victoriameter.NewMeter()
s := https.NewServer(server.Context(ctx), server.Address("127.0.0.1:0"), server.Codec("text/plain", codec.NewCodec()))
if err := s.Init(); err != nil {
t.Fatal(err)
}
if err := meterhandler.RegisterMeterServer(s, meterhandler.NewHandler()); err != nil {
t.Fatal(err)
}
if err := s.Start(); err != nil {
t.Fatal(err)
}
defer func() {
if err := s.Stop(); err != nil {
t.Fatal(err)
}
}()
segmentio.DefaultWriterConfig.Async = true
segmentio.DefaultWriterConfig.BatchTimeout = 1 * time.Second
segmentio.DefaultWriterConfig.RequiredAcks = int(kafka.RequireAll)
segmentio.DefaultReaderConfig.StartOffset = kafka.FirstOffset
segmentio.DefaultReaderConfig.MinBytes = 1024 * 10 // 10 kb
segmentio.DefaultReaderConfig.MaxBytes = 1024 * 1024 * 20 // 20 Mb
segmentio.DefaultReaderConfig.MaxWait = 20 * time.Second // 20s
segmentio.DefaultReaderConfig.QueueCapacity = 500
segmentio.DefaultReaderConfig.ReadBackoffMin = 2 * time.Second
segmentio.DefaultReaderConfig.ReadBackoffMax = 5 * time.Second
segmentio.DefaultReaderConfig.Logger = &lg{}
segmentio.DefaultReaderConfig.CommitInterval = 1 * time.Second
brk := segmentio.NewBroker(broker.Context(ctx), broker.Addrs(addrs...), segmentio.StatsInterval(5*time.Second),
segmentio.ClientID("test_sub"),
)
t.Logf("init")
if err := brk.Init(); err != nil {
t.Fatal(err)
}
t.Logf("connect")
if err := brk.Connect(ctx); err != nil {
t.Fatal(err)
}
defer func() {
t.Logf("disconnect")
if err := brk.Disconnect(ctx); err != nil {
t.Fatal(err)
}
}()
fmt.Printf("prefill topic\n")
go func() {
for i := 0; i < 900000; i++ {
if err := brk.Publish(ctx, topic, bm); err != nil {
t.Fatal(err)
}
//log.Printf("publish %d", i)
time.Sleep(200 * time.Millisecond)
}
}()
fmt.Printf("prefill complete\n")
var cnt uint64
var wait atomic.Value
wait.Store(true)
fn := func(msg broker.Event) error {
if wait.Load().(bool) {
wait.Store(false)
}
atomic.AddUint64(&cnt, 1)
return msg.Ack()
}
sub1, err := brk.Subscribe(ctx, topic, fn, broker.SubscribeGroup("test"), broker.SubscribeBodyOnly(true), broker.SubscribeAutoAck(true))
if err != nil {
t.Fatal(err)
}
sub2, err := brk.Subscribe(ctx, topic, fn, broker.SubscribeGroup("test"), broker.SubscribeBodyOnly(true), broker.SubscribeAutoAck(true))
if err != nil {
t.Fatal(err)
}
fmt.Printf("wait for ready\n")
for {
if !wait.Load().(bool) {
break
}
time.Sleep(1 * time.Second)
}
time.Sleep(5 * time.Second)
fmt.Printf("unsub\n")
if err := sub1.Unsubscribe(ctx); err != nil {
t.Fatal(err)
}
time.Sleep(9 * time.Second)
sub1, err = brk.Subscribe(ctx, topic, fn, broker.SubscribeGroup("test"), broker.SubscribeBodyOnly(true), broker.SubscribeAutoAck(true))
if err != nil {
t.Fatal(err)
}
t1 := time.NewTicker(10 * time.Second)
defer t1.Stop()
t2 := time.NewTicker(30 * time.Second)
defer t2.Stop()
for {
select {
case <-t1.C:
fmt.Printf("unsub from sub2\n")
if err := sub2.Unsubscribe(ctx); err != nil {
t.Fatal(err)
}
fmt.Printf("sub1\n")
sub1, err = brk.Subscribe(ctx, topic, fn, broker.SubscribeGroup("test"), broker.SubscribeBodyOnly(true), broker.SubscribeAutoAck(true))
if err != nil {
t.Fatal(err)
}
case <-t2.C:
fmt.Printf("unsub from sub1\n")
if err := sub1.Unsubscribe(ctx); err != nil {
t.Fatal(err)
}
fmt.Printf("sub2\n")
sub2, err = brk.Subscribe(ctx, topic, fn, broker.SubscribeGroup("test"), broker.SubscribeBodyOnly(true), broker.SubscribeAutoAck(true))
if err != nil {
t.Fatal(err)
}
}
}
}
func TestSub(t *testing.T) {
topic := fmt.Sprintf("test_topic")
if tr := os.Getenv("INTEGRATION_TESTS"); len(tr) > 0 {
@@ -106,8 +258,11 @@ func TestSub(t *testing.T) {
fmt.Printf("prefill topic\n")
go func() {
for i := 0; i < 900000; i++ {
// brk.Publish(ctx, topic, bm)
time.Sleep(1 * time.Second)
if err := brk.Publish(ctx, topic, bm); err != nil {
t.Fatal(err)
}
log.Printf("publish %d", i)
// time.Sleep(1 * time.Second)
}
}()
fmt.Printf("prefill complete\n")
@@ -200,7 +355,7 @@ func BenchmarkPub(b *testing.B) {
}
}()
segmentio.DefaultWriterConfig.Async = true
segmentio.DefaultWriterConfig.Async = false
segmentio.DefaultWriterConfig.BatchTimeout = 1 * time.Second
segmentio.DefaultWriterConfig.RequiredAcks = int(kafka.RequireAll)
fn := func(msgs []kafka.Message, err error) {