Secret implementation of config. Supporting config merge (#2027)

Co-authored-by: Asim Aslam <asim@aslam.me>

.github/workflows/pr.yml

@@ -25,4 +25,11 @@ jobs:
       id: tests
       env:
         IN_TRAVIS_CI: yes
-      run: go test -v ./...
+        S3_BLOB_STORE_REGION: ${{ secrets.SCALEWAY_REGION }}
+        S3_BLOB_STORE_ENDPOINT: ${{ secrets.SCALEWAY_ENDPOINT }}
+        S3_BLOB_STORE_ACCESS_KEY: ${{ secrets.SCALEWAY_ACCESS_KEY }}
+        S3_BLOB_STORE_SECRET_KEY: ${{ secrets.SCALEWAY_SECRET_KEY }}
+      run: |
+        wget -qO- https://binaries.cockroachdb.com/cockroach-v20.1.4.linux-amd64.tgz | tar  xvz
+        cockroach-v20.1.4.linux-amd64/cockroach start-single-node --insecure &
+        go test -v ./...

.github/workflows/tests.yml

@@ -7,13 +7,17 @@ jobs:
     name: Test repo
     runs-on: ubuntu-latest
     steps:
-
     - name: Set up Go 1.13
       uses: actions/setup-go@v1
       with:
         go-version: 1.13
       id: go
 
+    - name: Setup Kind
+      uses: engineerd/setup-kind@v0.4.0
+      with:
+        version: v0.8.1
+
     - name: Check out code into the Go module directory
       uses: actions/checkout@v2
 
@@ -25,30 +29,20 @@ jobs:
       id: tests
       env:
         IN_TRAVIS_CI: yes
+        S3_BLOB_STORE_REGION: ${{ secrets.SCALEWAY_REGION }}
+        S3_BLOB_STORE_ENDPOINT: ${{ secrets.SCALEWAY_ENDPOINT }}
+        S3_BLOB_STORE_ACCESS_KEY: ${{ secrets.SCALEWAY_ACCESS_KEY }}
+        S3_BLOB_STORE_SECRET_KEY: ${{ secrets.SCALEWAY_SECRET_KEY }}
       run: |
-        wget -qO- https://binaries.cockroachdb.com/cockroach-v20.1.4.linux-amd64.tgz | tar  xvz
+        kubectl apply -f runtime/kubernetes/test/test.yaml
+        sudo mkdir -p /var/run/secrets/kubernetes.io/serviceaccount
+        sudo chmod 777 /var/run/secrets/kubernetes.io/serviceaccount
+        wget -qO- https://binaries.cockroachdb.com/cockroach-v20.1.4.linux-amd64.tgz | tar -xvz
         cockroach-v20.1.4.linux-amd64/cockroach start-single-node --insecure &
-        go test -v ./...
+        wget -q https://github.com/nats-io/nats-streaming-server/releases/download/v0.18.0/nats-streaming-server-v0.18.0-linux-amd64.zip
+        unzip ./nats-streaming-server-v0.18.0-linux-amd64.zip
+        export PATH=$PATH:./nats-streaming-server-v0.18.0-linux-amd64
+        nats-streaming-server &
+        go test -tags kubernetes,nats -v ./...
   
-    - name: Notify of test failure
-      if: failure()
-      uses: rtCamp/action-slack-notify@v2.0.0
-      env:
-        SLACK_CHANNEL: build
-        SLACK_COLOR: '#BF280A'
-        SLACK_ICON: https://github.githubassets.com/images/modules/logos_page/GitHub-Mark.png
-        SLACK_TITLE: Tests Failed
-        SLACK_USERNAME: GitHub Actions
-        SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
-        
-    - name: Notify of test success
-      if: success()
-      uses: rtCamp/action-slack-notify@v2.0.0
-      env:
-        SLACK_CHANNEL: build
-        SLACK_COLOR: '#1FAD2B'
-        SLACK_ICON: https://github.githubassets.com/images/modules/logos_page/GitHub-Mark.png
-        SLACK_TITLE: Tests Passed
-        SLACK_USERNAME: GitHub Actions
-        SLACK_WEBHOOK: ${{ secrets.SLACK_WEBHOOK_URL }}
 

api/handler/rpc/rpc.go

@@ -249,7 +249,7 @@ func requestPayload(r *http.Request) ([]byte, error) {
 			return nil, err
 		}
 		return raw.Marshal()
-	case strings.Contains(ct, "application/www-x-form-urlencoded"):
+	case strings.Contains(ct, "application/x-www-form-urlencoded"):
 		r.ParseForm()
 
 		// generate a new set of values from the form
@@ -364,6 +364,13 @@ func requestPayload(r *http.Request) ([]byte, error) {
 			bodybuf = b
 		}
 		if bodydst == "" || bodydst == "*" {
+			// jsonpatch resequences the json object so we avoid it if possible (some usecases such as
+			// validating signatures require the request body to be unchangedd). We're keeping support
+			// for the custom paramaters for backwards compatability reasons.
+			if string(out) == "{}" {
+				return bodybuf, nil
+			}
+
 			if out, err = jsonpatch.MergeMergePatches(out, bodybuf); err == nil {
 				return out, nil
 			}
@@ -410,7 +417,6 @@ func requestPayload(r *http.Request) ([]byte, error) {
 
 		//fallback to previous unknown behaviour
 		return bodybuf, nil
-
 	}
 
 	return []byte{}, nil

api/server/cors/cors.go

@@ -40,5 +40,5 @@ func SetHeaders(w http.ResponseWriter, r *http.Request) {
 
 	set(w, "Access-Control-Allow-Credentials", "true")
 	set(w, "Access-Control-Allow-Methods", "POST, PATCH, GET, OPTIONS, PUT, DELETE")
-	set(w, "Access-Control-Allow-Headers", "Accept, Content-Type, Content-Length, Accept-Encoding, X-CSRF-Token, Authorization")
+	set(w, "Access-Control-Allow-Headers", "Accept, Content-Type, Content-Length, Accept-Encoding, X-CSRF-Token, Authorization, Namespace")
 }

api/service/proto/api.pb.go

@@ -1,268 +0,0 @@
-// Code generated by protoc-gen-go. DO NOT EDIT.
-// source: api/service/proto/api.proto
-
-package go_micro_api
-
-import (
-	context "context"
-	fmt "fmt"
-	proto "github.com/golang/protobuf/proto"
-	grpc "google.golang.org/grpc"
-	codes "google.golang.org/grpc/codes"
-	status "google.golang.org/grpc/status"
-	math "math"
-)
-
-// Reference imports to suppress errors if they are not otherwise used.
-var _ = proto.Marshal
-var _ = fmt.Errorf
-var _ = math.Inf
-
-// This is a compile-time assertion to ensure that this generated file
-// is compatible with the proto package it is being compiled against.
-// A compilation error at this line likely means your copy of the
-// proto package needs to be updated.
-const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package
-
-type Endpoint struct {
-	Name                 string   `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
-	Host                 []string `protobuf:"bytes,2,rep,name=host,proto3" json:"host,omitempty"`
-	Path                 []string `protobuf:"bytes,3,rep,name=path,proto3" json:"path,omitempty"`
-	Method               []string `protobuf:"bytes,4,rep,name=method,proto3" json:"method,omitempty"`
-	Stream               bool     `protobuf:"varint,5,opt,name=stream,proto3" json:"stream,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
-}
-
-func (m *Endpoint) Reset()         { *m = Endpoint{} }
-func (m *Endpoint) String() string { return proto.CompactTextString(m) }
-func (*Endpoint) ProtoMessage()    {}
-func (*Endpoint) Descriptor() ([]byte, []int) {
-	return fileDescriptor_c4a48b6b680b5c31, []int{0}
-}
-
-func (m *Endpoint) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_Endpoint.Unmarshal(m, b)
-}
-func (m *Endpoint) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_Endpoint.Marshal(b, m, deterministic)
-}
-func (m *Endpoint) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_Endpoint.Merge(m, src)
-}
-func (m *Endpoint) XXX_Size() int {
-	return xxx_messageInfo_Endpoint.Size(m)
-}
-func (m *Endpoint) XXX_DiscardUnknown() {
-	xxx_messageInfo_Endpoint.DiscardUnknown(m)
-}
-
-var xxx_messageInfo_Endpoint proto.InternalMessageInfo
-
-func (m *Endpoint) GetName() string {
-	if m != nil {
-		return m.Name
-	}
-	return ""
-}
-
-func (m *Endpoint) GetHost() []string {
-	if m != nil {
-		return m.Host
-	}
-	return nil
-}
-
-func (m *Endpoint) GetPath() []string {
-	if m != nil {
-		return m.Path
-	}
-	return nil
-}
-
-func (m *Endpoint) GetMethod() []string {
-	if m != nil {
-		return m.Method
-	}
-	return nil
-}
-
-func (m *Endpoint) GetStream() bool {
-	if m != nil {
-		return m.Stream
-	}
-	return false
-}
-
-type EmptyResponse struct {
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
-}
-
-func (m *EmptyResponse) Reset()         { *m = EmptyResponse{} }
-func (m *EmptyResponse) String() string { return proto.CompactTextString(m) }
-func (*EmptyResponse) ProtoMessage()    {}
-func (*EmptyResponse) Descriptor() ([]byte, []int) {
-	return fileDescriptor_c4a48b6b680b5c31, []int{1}
-}
-
-func (m *EmptyResponse) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_EmptyResponse.Unmarshal(m, b)
-}
-func (m *EmptyResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_EmptyResponse.Marshal(b, m, deterministic)
-}
-func (m *EmptyResponse) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_EmptyResponse.Merge(m, src)
-}
-func (m *EmptyResponse) XXX_Size() int {
-	return xxx_messageInfo_EmptyResponse.Size(m)
-}
-func (m *EmptyResponse) XXX_DiscardUnknown() {
-	xxx_messageInfo_EmptyResponse.DiscardUnknown(m)
-}
-
-var xxx_messageInfo_EmptyResponse proto.InternalMessageInfo
-
-func init() {
-	proto.RegisterType((*Endpoint)(nil), "go.micro.api.Endpoint")
-	proto.RegisterType((*EmptyResponse)(nil), "go.micro.api.EmptyResponse")
-}
-
-func init() { proto.RegisterFile("api/service/proto/api.proto", fileDescriptor_c4a48b6b680b5c31) }
-
-var fileDescriptor_c4a48b6b680b5c31 = []byte{
-	// 212 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xa4, 0xd0, 0xc1, 0x4a, 0x03, 0x31,
-	0x10, 0x80, 0x61, 0xd7, 0xad, 0x65, 0x1d, 0x14, 0x21, 0x87, 0x12, 0xec, 0x65, 0xd9, 0x53, 0x4f,
-	0x59, 0xd0, 0x27, 0x28, 0xda, 0x17, 0xd8, 0x37, 0x88, 0xed, 0xd0, 0x9d, 0x43, 0x32, 0x43, 0x32,
-	0x14, 0x7c, 0x08, 0xdf, 0x59, 0x12, 0x2b, 0x2c, 0x5e, 0xbd, 0xfd, 0xf3, 0x1d, 0x86, 0x61, 0x60,
-	0xeb, 0x85, 0xc6, 0x8c, 0xe9, 0x42, 0x47, 0x1c, 0x25, 0xb1, 0xf2, 0xe8, 0x85, 0x5c, 0x2d, 0xf3,
-	0x70, 0x66, 0x17, 0xe8, 0x98, 0xd8, 0x79, 0xa1, 0xe1, 0x02, 0xdd, 0x21, 0x9e, 0x84, 0x29, 0xaa,
-	0x31, 0xb0, 0x8a, 0x3e, 0xa0, 0x6d, 0xfa, 0x66, 0x77, 0x3f, 0xd5, 0x2e, 0x36, 0x73, 0x56, 0x7b,
-	0xdb, 0xb7, 0xc5, 0x4a, 0x17, 0x13, 0xaf, 0xb3, 0x6d, 0x7f, 0xac, 0xb4, 0xd9, 0xc0, 0x3a, 0xa0,
-	0xce, 0x7c, 0xb2, 0xab, 0xaa, 0xd7, 0xa9, 0x78, 0xd6, 0x84, 0x3e, 0xd8, 0xbb, 0xbe, 0xd9, 0x75,
-	0xd3, 0x75, 0x1a, 0x9e, 0xe0, 0xf1, 0x10, 0x44, 0x3f, 0x27, 0xcc, 0xc2, 0x31, 0xe3, 0xcb, 0x57,
-	0x03, 0xed, 0x5e, 0xc8, 0xec, 0xa1, 0x9b, 0xf0, 0x4c, 0x59, 0x31, 0x99, 0x8d, 0x5b, 0xde, 0xea,
-	0x7e, 0x0f, 0x7d, 0xde, 0xfe, 0xf1, 0xe5, 0xa2, 0xe1, 0xc6, 0xbc, 0x01, 0xbc, 0x63, 0xfa, 0xdf,
-	0x92, 0x8f, 0x75, 0xfd, 0xd6, 0xeb, 0x77, 0x00, 0x00, 0x00, 0xff, 0xff, 0x46, 0x62, 0x67, 0x30,
-	0x4c, 0x01, 0x00, 0x00,
-}
-
-// Reference imports to suppress errors if they are not otherwise used.
-var _ context.Context
-var _ grpc.ClientConn
-
-// This is a compile-time assertion to ensure that this generated file
-// is compatible with the grpc package it is being compiled against.
-const _ = grpc.SupportPackageIsVersion4
-
-// ApiClient is the client API for Api service.
-//
-// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream.
-type ApiClient interface {
-	Register(ctx context.Context, in *Endpoint, opts ...grpc.CallOption) (*EmptyResponse, error)
-	Deregister(ctx context.Context, in *Endpoint, opts ...grpc.CallOption) (*EmptyResponse, error)
-}
-
-type apiClient struct {
-	cc *grpc.ClientConn
-}
-
-func NewApiClient(cc *grpc.ClientConn) ApiClient {
-	return &apiClient{cc}
-}
-
-func (c *apiClient) Register(ctx context.Context, in *Endpoint, opts ...grpc.CallOption) (*EmptyResponse, error) {
-	out := new(EmptyResponse)
-	err := c.cc.Invoke(ctx, "/go.micro.api.Api/Register", in, out, opts...)
-	if err != nil {
-		return nil, err
-	}
-	return out, nil
-}
-
-func (c *apiClient) Deregister(ctx context.Context, in *Endpoint, opts ...grpc.CallOption) (*EmptyResponse, error) {
-	out := new(EmptyResponse)
-	err := c.cc.Invoke(ctx, "/go.micro.api.Api/Deregister", in, out, opts...)
-	if err != nil {
-		return nil, err
-	}
-	return out, nil
-}
-
-// ApiServer is the server API for Api service.
-type ApiServer interface {
-	Register(context.Context, *Endpoint) (*EmptyResponse, error)
-	Deregister(context.Context, *Endpoint) (*EmptyResponse, error)
-}
-
-// UnimplementedApiServer can be embedded to have forward compatible implementations.
-type UnimplementedApiServer struct {
-}
-
-func (*UnimplementedApiServer) Register(ctx context.Context, req *Endpoint) (*EmptyResponse, error) {
-	return nil, status.Errorf(codes.Unimplemented, "method Register not implemented")
-}
-func (*UnimplementedApiServer) Deregister(ctx context.Context, req *Endpoint) (*EmptyResponse, error) {
-	return nil, status.Errorf(codes.Unimplemented, "method Deregister not implemented")
-}
-
-func RegisterApiServer(s *grpc.Server, srv ApiServer) {
-	s.RegisterService(&_Api_serviceDesc, srv)
-}
-
-func _Api_Register_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
-	in := new(Endpoint)
-	if err := dec(in); err != nil {
-		return nil, err
-	}
-	if interceptor == nil {
-		return srv.(ApiServer).Register(ctx, in)
-	}
-	info := &grpc.UnaryServerInfo{
-		Server:     srv,
-		FullMethod: "/go.micro.api.Api/Register",
-	}
-	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
-		return srv.(ApiServer).Register(ctx, req.(*Endpoint))
-	}
-	return interceptor(ctx, in, info, handler)
-}
-
-func _Api_Deregister_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
-	in := new(Endpoint)
-	if err := dec(in); err != nil {
-		return nil, err
-	}
-	if interceptor == nil {
-		return srv.(ApiServer).Deregister(ctx, in)
-	}
-	info := &grpc.UnaryServerInfo{
-		Server:     srv,
-		FullMethod: "/go.micro.api.Api/Deregister",
-	}
-	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
-		return srv.(ApiServer).Deregister(ctx, req.(*Endpoint))
-	}
-	return interceptor(ctx, in, info, handler)
-}
-
-var _Api_serviceDesc = grpc.ServiceDesc{
-	ServiceName: "go.micro.api.Api",
-	HandlerType: (*ApiServer)(nil),
-	Methods: []grpc.MethodDesc{
-		{
-			MethodName: "Register",
-			Handler:    _Api_Register_Handler,
-		},
-		{
-			MethodName: "Deregister",
-			Handler:    _Api_Deregister_Handler,
-		},
-	},
-	Streams:  []grpc.StreamDesc{},
-	Metadata: "api/service/proto/api.proto",
-}

api/service/proto/api.pb.micro.go

@@ -1,110 +0,0 @@
-// Code generated by protoc-gen-micro. DO NOT EDIT.
-// source: api/service/proto/api.proto
-
-package go_micro_api
-
-import (
-	fmt "fmt"
-	proto "github.com/golang/protobuf/proto"
-	math "math"
-)
-
-import (
-	context "context"
-	api "github.com/micro/go-micro/v3/api"
-	client "github.com/micro/go-micro/v3/client"
-	server "github.com/micro/go-micro/v3/server"
-)
-
-// Reference imports to suppress errors if they are not otherwise used.
-var _ = proto.Marshal
-var _ = fmt.Errorf
-var _ = math.Inf
-
-// This is a compile-time assertion to ensure that this generated file
-// is compatible with the proto package it is being compiled against.
-// A compilation error at this line likely means your copy of the
-// proto package needs to be updated.
-const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package
-
-// Reference imports to suppress errors if they are not otherwise used.
-var _ api.Endpoint
-var _ context.Context
-var _ client.Option
-var _ server.Option
-
-// Api Endpoints for Api service
-
-func NewApiEndpoints() []*api.Endpoint {
-	return []*api.Endpoint{}
-}
-
-// Client API for Api service
-
-type ApiService interface {
-	Register(ctx context.Context, in *Endpoint, opts ...client.CallOption) (*EmptyResponse, error)
-	Deregister(ctx context.Context, in *Endpoint, opts ...client.CallOption) (*EmptyResponse, error)
-}
-
-type apiService struct {
-	c    client.Client
-	name string
-}
-
-func NewApiService(name string, c client.Client) ApiService {
-	return &apiService{
-		c:    c,
-		name: name,
-	}
-}
-
-func (c *apiService) Register(ctx context.Context, in *Endpoint, opts ...client.CallOption) (*EmptyResponse, error) {
-	req := c.c.NewRequest(c.name, "Api.Register", in)
-	out := new(EmptyResponse)
-	err := c.c.Call(ctx, req, out, opts...)
-	if err != nil {
-		return nil, err
-	}
-	return out, nil
-}
-
-func (c *apiService) Deregister(ctx context.Context, in *Endpoint, opts ...client.CallOption) (*EmptyResponse, error) {
-	req := c.c.NewRequest(c.name, "Api.Deregister", in)
-	out := new(EmptyResponse)
-	err := c.c.Call(ctx, req, out, opts...)
-	if err != nil {
-		return nil, err
-	}
-	return out, nil
-}
-
-// Server API for Api service
-
-type ApiHandler interface {
-	Register(context.Context, *Endpoint, *EmptyResponse) error
-	Deregister(context.Context, *Endpoint, *EmptyResponse) error
-}
-
-func RegisterApiHandler(s server.Server, hdlr ApiHandler, opts ...server.HandlerOption) error {
-	type api interface {
-		Register(ctx context.Context, in *Endpoint, out *EmptyResponse) error
-		Deregister(ctx context.Context, in *Endpoint, out *EmptyResponse) error
-	}
-	type Api struct {
-		api
-	}
-	h := &apiHandler{hdlr}
-	return s.Handle(s.NewHandler(&Api{h}, opts...))
-}
-
-type apiHandler struct {
-	ApiHandler
-}
-
-func (h *apiHandler) Register(ctx context.Context, in *Endpoint, out *EmptyResponse) error {
-	return h.ApiHandler.Register(ctx, in, out)
-}
-
-func (h *apiHandler) Deregister(ctx context.Context, in *Endpoint, out *EmptyResponse) error {
-	return h.ApiHandler.Deregister(ctx, in, out)
-}

api/service/proto/api.proto

@@ -1,18 +0,0 @@
-syntax = "proto3";
-
-package go.micro.api;
-
-service Api {
-  rpc Register(Endpoint) returns (EmptyResponse) {};
-  rpc Deregister(Endpoint) returns (EmptyResponse) {};
-}
-
-message Endpoint {
-  string name = 1;
-  repeated string host = 2;
-  repeated string path = 3;
-  repeated string method = 4;
-  bool stream = 5;
-}
-
-message EmptyResponse {}

auth/auth.go

@@ -2,14 +2,11 @@
 package auth
 
 import (
-	"context"
 	"errors"
 	"time"
 )
 
 const (
-	// BearerScheme used for Authorization header
-	BearerScheme = "Bearer "
 	// ScopePublic is the scope applied to a rule to allow access to the public
 	ScopePublic = ""
 	// ScopeAccount is the scope applied to a rule to limit to users with any valid account
@@ -49,7 +46,7 @@ type Auth interface {
 
 // Account provided by an auth provider
 type Account struct {
-	// ID of the account e.g. email
+	// ID of the account e.g. UUID. Should not change
 	ID string `json:"id"`
 	// Type of the account, e.g. service
 	Type string `json:"type"`
@@ -61,6 +58,8 @@ type Account struct {
 	Scopes []string `json:"scopes"`
 	// Secret for the account, e.g. the password
 	Secret string `json:"secret"`
+	// Name of the account. User friendly name that might change e.g. a username or email
+	Name string `json:"name"`
 }
 
 // Token can be short or long lived
@@ -115,19 +114,3 @@ type Rule struct {
 	// rule will be applied
 	Priority int32
 }
-
-type accountKey struct{}
-
-// AccountFromContext gets the account from the context, which
-// is set by the auth wrapper at the start of a call. If the account
-// is not set, a nil account will be returned. The error is only returned
-// when there was a problem retrieving an account
-func AccountFromContext(ctx context.Context) (*Account, bool) {
-	acc, ok := ctx.Value(accountKey{}).(*Account)
-	return acc, ok
-}
-
-// ContextWithAccount sets the account in the context
-func ContextWithAccount(ctx context.Context, account *Account) context.Context {
-	return context.WithValue(ctx, accountKey{}, account)
-}

auth/jwt/jwt.go

@@ -54,13 +54,17 @@ func (j *jwtAuth) Generate(id string, opts ...auth.GenerateOption) (*auth.Accoun
 	if len(options.Issuer) == 0 {
 		options.Issuer = j.Options().Issuer
 	}
-
+	name := options.Name
+	if name == "" {
+		name = id
+	}
 	account := &auth.Account{
 		ID:       id,
 		Type:     options.Type,
 		Scopes:   options.Scopes,
 		Metadata: options.Metadata,
 		Issuer:   options.Issuer,
+		Name:     name,
 	}
 
 	// generate a JWT secret which can be provided to the Token() method

auth/noop/noop.go

@@ -40,13 +40,17 @@ func (n *noop) Options() auth.Options {
 // Generate a new account
 func (n *noop) Generate(id string, opts ...auth.GenerateOption) (*auth.Account, error) {
 	options := auth.NewGenerateOptions(opts...)
-
+	name := options.Name
+	if name == "" {
+		name = id
+	}
 	return &auth.Account{
 		ID:       id,
 		Secret:   options.Secret,
 		Metadata: options.Metadata,
 		Scopes:   options.Scopes,
 		Issuer:   n.Options().Issuer,
+		Name:     name,
 	}, nil
 }
 

auth/options.go

@@ -110,6 +110,8 @@ type GenerateOptions struct {
 	Secret string
 	// Issuer of the account, e.g. micro
 	Issuer string
+	// Name of the acouunt e.g. an email or username
+	Name string
 }
 
 type GenerateOption func(o *GenerateOptions)
@@ -156,6 +158,13 @@ func WithIssuer(i string) GenerateOption {
 	}
 }
 
+// WithName for the generated account
+func WithName(n string) GenerateOption {
+	return func(o *GenerateOptions) {
+		o.Name = n
+	}
+}
+
 // NewGenerateOptions from a slice of options
 func NewGenerateOptions(opts ...GenerateOption) GenerateOptions {
 	var options GenerateOptions

broker/nats/context.go

@@ -1,17 +0,0 @@
-package nats
-
-import (
-	"context"
-
-	"github.com/micro/go-micro/v3/broker"
-)
-
-// setBrokerOption returns a function to setup a context with given value
-func setBrokerOption(k, v interface{}) broker.Option {
-	return func(o *broker.Options) {
-		if o.Context == nil {
-			o.Context = context.Background()
-		}
-		o.Context = context.WithValue(o.Context, k, v)
-	}
-}

broker/nats/nats.go

@@ -1,294 +0,0 @@
-// Package nats provides a NATS broker
-package nats
-
-import (
-	"context"
-	"errors"
-	"strings"
-	"sync"
-
-	"github.com/micro/go-micro/v3/broker"
-	"github.com/micro/go-micro/v3/codec/json"
-	"github.com/micro/go-micro/v3/logger"
-	"github.com/micro/go-micro/v3/registry/mdns"
-	nats "github.com/nats-io/nats.go"
-)
-
-type natsBroker struct {
-	sync.Once
-	sync.RWMutex
-
-	// indicate if we're connected
-	connected bool
-
-	addrs []string
-	conn  *nats.Conn
-	opts  broker.Options
-	nopts nats.Options
-
-	// should we drain the connection
-	drain   bool
-	closeCh chan (error)
-}
-
-type subscriber struct {
-	s    *nats.Subscription
-	opts broker.SubscribeOptions
-}
-
-func (s *subscriber) Options() broker.SubscribeOptions {
-	return s.opts
-}
-
-func (s *subscriber) Topic() string {
-	return s.s.Subject
-}
-
-func (s *subscriber) Unsubscribe() error {
-	return s.s.Unsubscribe()
-}
-
-func (n *natsBroker) Address() string {
-	if n.conn != nil && n.conn.IsConnected() {
-		return n.conn.ConnectedUrl()
-	}
-
-	if len(n.addrs) > 0 {
-		return n.addrs[0]
-	}
-
-	return ""
-}
-
-func (n *natsBroker) setAddrs(addrs []string) []string {
-	//nolint:prealloc
-	var cAddrs []string
-	for _, addr := range addrs {
-		if len(addr) == 0 {
-			continue
-		}
-		if !strings.HasPrefix(addr, "nats://") {
-			addr = "nats://" + addr
-		}
-		cAddrs = append(cAddrs, addr)
-	}
-	if len(cAddrs) == 0 {
-		cAddrs = []string{nats.DefaultURL}
-	}
-	return cAddrs
-}
-
-func (n *natsBroker) Connect() error {
-	n.Lock()
-	defer n.Unlock()
-
-	if n.connected {
-		return nil
-	}
-
-	status := nats.CLOSED
-	if n.conn != nil {
-		status = n.conn.Status()
-	}
-
-	switch status {
-	case nats.CONNECTED, nats.RECONNECTING, nats.CONNECTING:
-		n.connected = true
-		return nil
-	default: // DISCONNECTED or CLOSED or DRAINING
-		opts := n.nopts
-		opts.Servers = n.addrs
-		opts.Secure = n.opts.Secure
-		opts.TLSConfig = n.opts.TLSConfig
-
-		// secure might not be set
-		if n.opts.TLSConfig != nil {
-			opts.Secure = true
-		}
-
-		c, err := opts.Connect()
-		if err != nil {
-			if logger.V(logger.WarnLevel, logger.DefaultLogger) {
-				logger.Warnf("Error connecting to broker: %v", err)
-			}
-
-			return err
-		}
-		n.conn = c
-		n.connected = true
-		return nil
-	}
-}
-
-func (n *natsBroker) Disconnect() error {
-	n.Lock()
-	defer n.Unlock()
-
-	// drain the connection if specified
-	if n.drain {
-		n.conn.Drain()
-		n.closeCh <- nil
-	}
-
-	// close the client connection
-	n.conn.Close()
-
-	// set not connected
-	n.connected = false
-
-	return nil
-}
-
-func (n *natsBroker) Init(opts ...broker.Option) error {
-	n.setOption(opts...)
-	return nil
-}
-
-func (n *natsBroker) Options() broker.Options {
-	return n.opts
-}
-
-func (n *natsBroker) Publish(topic string, msg *broker.Message, opts ...broker.PublishOption) error {
-	n.RLock()
-	defer n.RUnlock()
-
-	if n.conn == nil {
-		return errors.New("not connected")
-	}
-
-	b, err := n.opts.Codec.Marshal(msg)
-	if err != nil {
-		return err
-	}
-	return n.conn.Publish(topic, b)
-}
-
-func (n *natsBroker) Subscribe(topic string, handler broker.Handler, opts ...broker.SubscribeOption) (broker.Subscriber, error) {
-	n.RLock()
-	if n.conn == nil {
-		n.RUnlock()
-		return nil, errors.New("not connected")
-	}
-	n.RUnlock()
-
-	opt := broker.SubscribeOptions{
-		Context: context.Background(),
-	}
-
-	for _, o := range opts {
-		o(&opt)
-	}
-
-	fn := func(msg *nats.Msg) {
-		var m *broker.Message
-		eh := opt.ErrorHandler
-		err := n.opts.Codec.Unmarshal(msg.Data, &m)
-		if err != nil {
-			m.Body = msg.Data
-			if logger.V(logger.ErrorLevel, logger.DefaultLogger) {
-				logger.Error(err)
-			}
-			if eh != nil {
-				eh(m, err)
-			}
-			return
-		}
-		if err := handler(m); err != nil {
-			if logger.V(logger.ErrorLevel, logger.DefaultLogger) {
-				logger.Error(err)
-			}
-			if eh != nil {
-				eh(m, err)
-			}
-		}
-	}
-
-	var sub *nats.Subscription
-	var err error
-
-	n.RLock()
-	if len(opt.Queue) > 0 {
-		sub, err = n.conn.QueueSubscribe(topic, opt.Queue, fn)
-	} else {
-		sub, err = n.conn.Subscribe(topic, fn)
-	}
-	n.RUnlock()
-	if err != nil {
-		return nil, err
-	}
-	return &subscriber{s: sub, opts: opt}, nil
-}
-
-func (n *natsBroker) String() string {
-	return "nats"
-}
-
-func (n *natsBroker) setOption(opts ...broker.Option) {
-	for _, o := range opts {
-		o(&n.opts)
-	}
-
-	n.Once.Do(func() {
-		n.nopts = nats.GetDefaultOptions()
-	})
-
-	if nopts, ok := n.opts.Context.Value(optionsKey{}).(nats.Options); ok {
-		n.nopts = nopts
-	}
-
-	// broker.Options have higher priority than nats.Options
-	// only if Addrs, Secure or TLSConfig were not set through a broker.Option
-	// we read them from nats.Option
-	if len(n.opts.Addrs) == 0 {
-		n.opts.Addrs = n.nopts.Servers
-	}
-
-	if !n.opts.Secure {
-		n.opts.Secure = n.nopts.Secure
-	}
-
-	if n.opts.TLSConfig == nil {
-		n.opts.TLSConfig = n.nopts.TLSConfig
-	}
-	n.addrs = n.setAddrs(n.opts.Addrs)
-
-	if n.opts.Context.Value(drainConnectionKey{}) != nil {
-		n.drain = true
-		n.closeCh = make(chan error)
-		n.nopts.ClosedCB = n.onClose
-		n.nopts.AsyncErrorCB = n.onAsyncError
-		n.nopts.DisconnectedErrCB = n.onDisconnectedError
-	}
-}
-
-func (n *natsBroker) onClose(conn *nats.Conn) {
-	n.closeCh <- nil
-}
-
-func (n *natsBroker) onAsyncError(conn *nats.Conn, sub *nats.Subscription, err error) {
-	// There are kinds of different async error nats might callback, but we are interested
-	// in ErrDrainTimeout only here.
-	if err == nats.ErrDrainTimeout {
-		n.closeCh <- err
-	}
-}
-
-func (n *natsBroker) onDisconnectedError(conn *nats.Conn, err error) {
-	n.closeCh <- err
-}
-
-func NewBroker(opts ...broker.Option) broker.Broker {
-	options := broker.Options{
-		// Default codec
-		Codec:    json.Marshaler{},
-		Context:  context.Background(),
-		Registry: mdns.NewRegistry(),
-	}
-
-	n := &natsBroker{
-		opts: options,
-	}
-	n.setOption(opts...)
-
-	return n
-}

broker/nats/nats_test.go

@@ -1,98 +0,0 @@
-package nats
-
-import (
-	"fmt"
-	"testing"
-
-	"github.com/micro/go-micro/v3/broker"
-	nats "github.com/nats-io/nats.go"
-)
-
-var addrTestCases = []struct {
-	name        string
-	description string
-	addrs       map[string]string // expected address : set address
-}{
-	{
-		"brokerOpts",
-		"set broker addresses through a broker.Option in constructor",
-		map[string]string{
-			"nats://192.168.10.1:5222": "192.168.10.1:5222",
-			"nats://10.20.10.0:4222":   "10.20.10.0:4222"},
-	},
-	{
-		"brokerInit",
-		"set broker addresses through a broker.Option in broker.Init()",
-		map[string]string{
-			"nats://192.168.10.1:5222": "192.168.10.1:5222",
-			"nats://10.20.10.0:4222":   "10.20.10.0:4222"},
-	},
-	{
-		"natsOpts",
-		"set broker addresses through the nats.Option in constructor",
-		map[string]string{
-			"nats://192.168.10.1:5222": "192.168.10.1:5222",
-			"nats://10.20.10.0:4222":   "10.20.10.0:4222"},
-	},
-	{
-		"default",
-		"check if default Address is set correctly",
-		map[string]string{
-			"nats://127.0.0.1:4222": "",
-		},
-	},
-}
-
-// TestInitAddrs tests issue #100. Ensures that if the addrs is set by an option in init it will be used.
-func TestInitAddrs(t *testing.T) {
-
-	for _, tc := range addrTestCases {
-		t.Run(fmt.Sprintf("%s: %s", tc.name, tc.description), func(t *testing.T) {
-
-			var br broker.Broker
-			var addrs []string
-
-			for _, addr := range tc.addrs {
-				addrs = append(addrs, addr)
-			}
-
-			switch tc.name {
-			case "brokerOpts":
-				// we know that there are just two addrs in the dict
-				br = NewBroker(broker.Addrs(addrs[0], addrs[1]))
-				br.Init()
-			case "brokerInit":
-				br = NewBroker()
-				// we know that there are just two addrs in the dict
-				br.Init(broker.Addrs(addrs[0], addrs[1]))
-			case "natsOpts":
-				nopts := nats.GetDefaultOptions()
-				nopts.Servers = addrs
-				br = NewBroker(Options(nopts))
-				br.Init()
-			case "default":
-				br = NewBroker()
-				br.Init()
-			}
-
-			natsBroker, ok := br.(*natsBroker)
-			if !ok {
-				t.Fatal("Expected broker to be of types *natsBroker")
-			}
-			// check if the same amount of addrs we set has actually been set, default
-			// have only 1 address nats://127.0.0.1:4222 (current nats code) or
-			// nats://localhost:4222 (older code version)
-			if len(natsBroker.addrs) != len(tc.addrs) && tc.name != "default" {
-				t.Errorf("Expected Addr count = %d, Actual Addr count = %d",
-					len(natsBroker.addrs), len(tc.addrs))
-			}
-
-			for _, addr := range natsBroker.addrs {
-				_, ok := tc.addrs[addr]
-				if !ok {
-					t.Errorf("Expected '%s' has not been set", addr)
-				}
-			}
-		})
-	}
-}

broker/nats/options.go

@@ -1,19 +0,0 @@
-package nats
-
-import (
-	"github.com/micro/go-micro/v3/broker"
-	nats "github.com/nats-io/nats.go"
-)
-
-type optionsKey struct{}
-type drainConnectionKey struct{}
-
-// Options accepts nats.Options
-func Options(opts nats.Options) broker.Option {
-	return setBrokerOption(optionsKey{}, opts)
-}
-
-// DrainConnection will drain subscription on close
-func DrainConnection() broker.Option {
-	return setBrokerOption(drainConnectionKey{}, struct{}{})
-}

cache/memcache/memcache.go

@@ -1,80 +0,0 @@
-// Package memcache is a memcache implementation of the Cache
-package memcache
-
-import (
-	"encoding/json"
-
-	"github.com/bradfitz/gomemcache/memcache"
-	"github.com/micro/go-micro/v3/cache"
-)
-
-type memcacheCache struct {
-	options cache.Options
-	client  *memcache.Client
-}
-
-type memcacheItem struct {
-	Key   string
-	Value interface{}
-}
-
-func (m *memcacheCache) Init(opts ...cache.Option) error {
-	for _, o := range opts {
-		o(&m.options)
-	}
-	return nil
-}
-
-func (m *memcacheCache) Get(key string) (interface{}, error) {
-	item, err := m.client.Get(key)
-	if err != nil {
-		return nil, err
-	}
-
-	var mc *memcacheItem
-
-	if err := json.Unmarshal(item.Value, &mc); err != nil {
-		return nil, err
-	}
-
-	return mc.Value, nil
-}
-
-func (m *memcacheCache) Set(key string, val interface{}) error {
-	b, err := json.Marshal(val)
-	if err != nil {
-		return err
-	}
-
-	return m.client.Set(&memcache.Item{
-		Key:   key,
-		Value: b,
-	})
-}
-
-func (m *memcacheCache) Delete(key string) error {
-	return m.client.Delete(key)
-}
-
-func (m *memcacheCache) String() string {
-	return "memcache"
-}
-
-// NewCache returns a new memcache Cache
-func NewCache(opts ...cache.Option) cache.Cache {
-	var options cache.Options
-	for _, o := range opts {
-		o(&options)
-	}
-
-	// get and set the nodes
-	nodes := options.Nodes
-	if len(nodes) == 0 {
-		nodes = []string{"localhost:11211"}
-	}
-
-	return &memcacheCache{
-		options: options,
-		client:  memcache.New(nodes...),
-	}
-}

client/backoff_test.go

@@ -4,6 +4,8 @@ import (
 	"context"
 	"testing"
 	"time"
+
+	"github.com/micro/go-micro/v3/codec"
 )
 
 func TestBackoff(t *testing.T) {
@@ -32,3 +34,63 @@ func TestBackoff(t *testing.T) {
 		}
 	}
 }
+
+type testRequest struct {
+	service     string
+	method      string
+	endpoint    string
+	contentType string
+	codec       codec.Codec
+	body        interface{}
+	opts        RequestOptions
+}
+
+func newRequest(service, endpoint string, request interface{}, contentType string, reqOpts ...RequestOption) Request {
+	var opts RequestOptions
+
+	for _, o := range reqOpts {
+		o(&opts)
+	}
+
+	// set the content-type specified
+	if len(opts.ContentType) > 0 {
+		contentType = opts.ContentType
+	}
+
+	return &testRequest{
+		service:     service,
+		method:      endpoint,
+		endpoint:    endpoint,
+		body:        request,
+		contentType: contentType,
+		opts:        opts,
+	}
+}
+
+func (r *testRequest) ContentType() string {
+	return r.contentType
+}
+
+func (r *testRequest) Service() string {
+	return r.service
+}
+
+func (r *testRequest) Method() string {
+	return r.method
+}
+
+func (r *testRequest) Endpoint() string {
+	return r.endpoint
+}
+
+func (r *testRequest) Body() interface{} {
+	return r.body
+}
+
+func (r *testRequest) Codec() codec.Writer {
+	return r.codec
+}
+
+func (r *testRequest) Stream() bool {
+	return r.opts.Stream
+}

client/cache.go

@@ -1,66 +0,0 @@
-package client
-
-import (
-	"context"
-	"encoding/json"
-	"fmt"
-	"hash/fnv"
-	"time"
-
-	"github.com/micro/go-micro/v3/metadata"
-	cache "github.com/patrickmn/go-cache"
-)
-
-// NewCache returns an initialised cache.
-func NewCache() *Cache {
-	return &Cache{
-		cache: cache.New(cache.NoExpiration, 30*time.Second),
-	}
-}
-
-// Cache for responses
-type Cache struct {
-	cache *cache.Cache
-}
-
-// Get a response from the cache
-func (c *Cache) Get(ctx context.Context, req Request) (interface{}, bool) {
-	return c.cache.Get(key(ctx, req))
-}
-
-// Set a response in the cache
-func (c *Cache) Set(ctx context.Context, req Request, rsp interface{}, expiry time.Duration) {
-	c.cache.Set(key(ctx, req), rsp, expiry)
-}
-
-// List the key value pairs in the cache
-func (c *Cache) List() map[string]string {
-	items := c.cache.Items()
-
-	rsp := make(map[string]string, len(items))
-	for k, v := range items {
-		bytes, _ := json.Marshal(v.Object)
-		rsp[k] = string(bytes)
-	}
-
-	return rsp
-}
-
-// key returns a hash for the context and request
-func key(ctx context.Context, req Request) string {
-	ns, _ := metadata.Get(ctx, "Micro-Namespace")
-
-	bytes, _ := json.Marshal(map[string]interface{}{
-		"namespace": ns,
-		"request": map[string]interface{}{
-			"service":  req.Service(),
-			"endpoint": req.Endpoint(),
-			"method":   req.Method(),
-			"body":     req.Body(),
-		},
-	})
-
-	h := fnv.New64()
-	h.Write(bytes)
-	return fmt.Sprintf("%x", h.Sum(nil))
-}

client/cache_test.go

@@ -1,77 +0,0 @@
-package client
-
-import (
-	"context"
-	"testing"
-	"time"
-
-	"github.com/micro/go-micro/v3/metadata"
-)
-
-func TestCache(t *testing.T) {
-	ctx := context.TODO()
-	req := &testRequest{service: "go.micro.service.foo", method: "Foo.Bar"}
-
-	t.Run("CacheMiss", func(t *testing.T) {
-		if _, ok := NewCache().Get(ctx, req); ok {
-			t.Errorf("Expected to get no result from Get")
-		}
-	})
-
-	t.Run("CacheHit", func(t *testing.T) {
-		c := NewCache()
-
-		rsp := "theresponse"
-		c.Set(ctx, req, rsp, time.Minute)
-
-		if res, ok := c.Get(ctx, req); !ok {
-			t.Errorf("Expected a result, got nothing")
-		} else if res != rsp {
-			t.Errorf("Expected '%v' result, got '%v'", rsp, res)
-		}
-	})
-}
-
-func TestCacheKey(t *testing.T) {
-	ctx := context.TODO()
-
-	req1 := &testRequest{service: "go.micro.service.foo", method: "Foo.Bar"}
-	req2 := &testRequest{service: "go.micro.service.foo", method: "Foo.Baz"}
-	req3 := &testRequest{service: "go.micro.service.foo", method: "Foo.Bar", body: "customquery"}
-
-	t.Run("IdenticalRequests", func(t *testing.T) {
-		key1 := key(ctx, req1)
-		key2 := key(ctx, req1)
-		if key1 != key2 {
-			t.Errorf("Expected the keys to match for identical requests and context")
-		}
-	})
-
-	t.Run("DifferentRequestEndpoints", func(t *testing.T) {
-		key1 := key(ctx, req1)
-		key2 := key(ctx, req2)
-
-		if key1 == key2 {
-			t.Errorf("Expected the keys to differ for different request endpoints")
-		}
-	})
-
-	t.Run("DifferentRequestBody", func(t *testing.T) {
-		key1 := key(ctx, req2)
-		key2 := key(ctx, req3)
-
-		if key1 == key2 {
-			t.Errorf("Expected the keys to differ for different request bodies")
-		}
-	})
-
-	t.Run("DifferentMetadata", func(t *testing.T) {
-		mdCtx := metadata.Set(context.TODO(), "Micro-Namespace", "bar")
-		key1 := key(mdCtx, req1)
-		key2 := key(ctx, req1)
-
-		if key1 == key2 {
-			t.Errorf("Expected the keys to differ for different metadata")
-		}
-	})
-}

client/context.go

@@ -1,16 +0,0 @@
-package client
-
-import (
-	"context"
-)
-
-type clientKey struct{}
-
-func FromContext(ctx context.Context) (Client, bool) {
-	c, ok := ctx.Value(clientKey{}).(Client)
-	return c, ok
-}
-
-func NewContext(ctx context.Context, c Client) context.Context {
-	return context.WithValue(ctx, clientKey{}, c)
-}

client/grpc/grpc.go

@@ -172,15 +172,6 @@ func (g *grpcClient) stream(ctx context.Context, addr string, req client.Request
 		return errors.InternalServerError("go.micro.client", err.Error())
 	}
 
-	var dialCtx context.Context
-	var cancel context.CancelFunc
-	if opts.DialTimeout >= 0 {
-		dialCtx, cancel = context.WithTimeout(ctx, opts.DialTimeout)
-	} else {
-		dialCtx, cancel = context.WithCancel(ctx)
-	}
-	defer cancel()
-
 	wc := wrapCodec{cf}
 
 	grpcDialOptions := []grpc.DialOption{
@@ -192,7 +183,7 @@ func (g *grpcClient) stream(ctx context.Context, addr string, req client.Request
 		grpcDialOptions = append(grpcDialOptions, opts...)
 	}
 
-	cc, err := grpc.DialContext(dialCtx, addr, grpcDialOptions...)
+	cc, err := g.pool.getConn(addr, grpcDialOptions...)
 	if err != nil {
 		return errors.InternalServerError("go.micro.client", fmt.Sprintf("Error sending request: %v", err))
 	}
@@ -211,16 +202,16 @@ func (g *grpcClient) stream(ctx context.Context, addr string, req client.Request
 		grpcCallOptions = append(grpcCallOptions, opts...)
 	}
 
-	// create a new cancelling context
+	var cancel context.CancelFunc
-	newCtx, cancel := context.WithCancel(ctx)
+	ctx, cancel = context.WithCancel(ctx)
 
-	st, err := cc.NewStream(newCtx, desc, methodToGRPC(req.Service(), req.Endpoint()), grpcCallOptions...)
+	st, err := cc.NewStream(ctx, desc, methodToGRPC(req.Service(), req.Endpoint()), grpcCallOptions...)
 	if err != nil {
 		// we need to cleanup as we dialled and created a context
 		// cancel the context
 		cancel()
-		// close the connection
+		// release the connection
-		cc.Close()
+		g.pool.release(addr, cc, err)
 		// now return the error
 		return errors.InternalServerError("go.micro.client", fmt.Sprintf("Error creating stream: %v", err))
 	}
@@ -246,8 +237,16 @@ func (g *grpcClient) stream(ctx context.Context, addr string, req client.Request
 			codec:  cf,
 			gcodec: codec,
 		},
-		conn:   cc,
+		conn: cc,
-		cancel: cancel,
+		close: func(err error) {
+			// cancel the context if an error occured
+			if err != nil {
+				cancel()
+			}
+
+			// defer execution of release
+			g.pool.release(addr, cc, err)
+		},
 	}
 
 	// set the stream as the response

client/grpc/response.go

@@ -10,7 +10,7 @@ import (
 )
 
 type response struct {
-	conn   *grpc.ClientConn
+	conn   *poolConn
 	stream grpc.ClientStream
 	codec  encoding.Codec
 	gcodec codec.Codec

client/grpc/stream.go

@@ -17,11 +17,11 @@ type grpcStream struct {
 	sync.RWMutex
 	closed   bool
 	err      error
-	conn     *grpc.ClientConn
+	conn     *poolConn
 	request  client.Request
 	response client.Response
 	context  context.Context
-	cancel   func()
+	close    func(err error)
 }
 
 func (g *grpcStream) Context() context.Context {
@@ -86,9 +86,9 @@ func (g *grpcStream) Close() error {
 	if g.closed {
 		return nil
 	}
-	// cancel the context
+
-	g.cancel()
+	// close the connection
 	g.closed = true
-	g.ClientStream.CloseSend()
+	g.close(g.err)
-	return g.conn.Close()
+	return g.ClientStream.CloseSend()
 }

client/lookup.go

@@ -19,16 +19,16 @@ func LookupRoute(ctx context.Context, req Request, opts CallOptions) ([]string,
 	}
 
 	// construct the router query
-	query := []router.QueryOption{router.QueryService(req.Service())}
+	query := []router.LookupOption{}
 
 	// if a custom network was requested, pass this to the router. By default the router will use it's
 	// own network, which is set during initialisation.
 	if len(opts.Network) > 0 {
-		query = append(query, router.QueryNetwork(opts.Network))
+		query = append(query, router.LookupNetwork(opts.Network))
 	}
 
 	// lookup the routes which can be used to execute the request
-	routes, err := opts.Router.Lookup(query...)
+	routes, err := opts.Router.Lookup(req.Service(), query...)
 	if err == router.ErrRouteNotFound {
 		return nil, errors.InternalServerError("go.micro.client", "service %s: %s", req.Service(), err.Error())
 	} else if err != nil {

client/mucp/mucp.go

@@ -1,4 +1,4 @@
-// Package mucp provides an mucp client
+// Package mucp provides a transport agnostic RPC client
 package mucp
 
 import (
@@ -14,7 +14,7 @@ import (
 	raw "github.com/micro/go-micro/v3/codec/bytes"
 	"github.com/micro/go-micro/v3/errors"
 	"github.com/micro/go-micro/v3/metadata"
-	"github.com/micro/go-micro/v3/transport"
+	"github.com/micro/go-micro/v3/network/transport"
 	"github.com/micro/go-micro/v3/util/buf"
 	"github.com/micro/go-micro/v3/util/pool"
 )

client/mucp/mucp_codec.go

@@ -12,8 +12,8 @@ import (
 	"github.com/micro/go-micro/v3/codec/proto"
 	"github.com/micro/go-micro/v3/codec/protorpc"
 	"github.com/micro/go-micro/v3/errors"
+	"github.com/micro/go-micro/v3/network/transport"
 	"github.com/micro/go-micro/v3/registry"
-	"github.com/micro/go-micro/v3/transport"
 )
 
 const (

client/mucp/mucp_response.go

@@ -2,7 +2,7 @@ package mucp
 
 import (
 	"github.com/micro/go-micro/v3/codec"
-	"github.com/micro/go-micro/v3/transport"
+	"github.com/micro/go-micro/v3/network/transport"
 )
 
 type rpcResponse struct {

client/mucp/options_test.go

@@ -5,7 +5,7 @@ import (
 	"time"
 
 	"github.com/micro/go-micro/v3/client"
-	"github.com/micro/go-micro/v3/transport"
+	"github.com/micro/go-micro/v3/network/transport"
 )
 
 func TestCallOptions(t *testing.T) {

client/options.go

@@ -7,13 +7,13 @@ import (
 	"github.com/micro/go-micro/v3/broker"
 	"github.com/micro/go-micro/v3/broker/http"
 	"github.com/micro/go-micro/v3/codec"
+	"github.com/micro/go-micro/v3/network/transport"
+	thttp "github.com/micro/go-micro/v3/network/transport/http"
 	"github.com/micro/go-micro/v3/registry"
 	"github.com/micro/go-micro/v3/router"
 	regRouter "github.com/micro/go-micro/v3/router/registry"
 	"github.com/micro/go-micro/v3/selector"
-	"github.com/micro/go-micro/v3/selector/random"
+	"github.com/micro/go-micro/v3/selector/roundrobin"
-	"github.com/micro/go-micro/v3/transport"
-	thttp "github.com/micro/go-micro/v3/transport/http"
 )
 
 type Options struct {
@@ -36,9 +36,6 @@ type Options struct {
 	PoolSize int
 	PoolTTL  time.Duration
 
-	// Response cache
-	Cache *Cache
-
 	// Middleware for client
 	Wrappers []Wrapper
 
@@ -55,8 +52,6 @@ type CallOptions struct {
 	Address []string
 	// Backoff func
 	Backoff BackoffFunc
-	// Duration to cache the response for
-	CacheExpiry time.Duration
 	// Transport Dial Timeout
 	DialTimeout time.Duration
 	// Number of Call attempts
@@ -109,7 +104,6 @@ type RequestOptions struct {
 
 func NewOptions(options ...Option) Options {
 	opts := Options{
-		Cache:       NewCache(),
 		Context:     context.Background(),
 		ContentType: "application/protobuf",
 		Codecs:      make(map[string]codec.NewCodec),
@@ -125,7 +119,7 @@ func NewOptions(options ...Option) Options {
 		PoolTTL:   DefaultPoolTTL,
 		Broker:    http.NewBroker(),
 		Router:    regRouter.NewRouter(),
-		Selector:  random.NewSelector(),
+		Selector:  roundrobin.NewSelector(),
 		Transport: thttp.NewTransport(),
 	}
 
@@ -357,14 +351,6 @@ func WithAuthToken() CallOption {
 	}
 }
 
-// WithCache is a CallOption which sets the duration the response
-// shoull be cached for
-func WithCache(c time.Duration) CallOption {
-	return func(o *CallOptions) {
-		o.CacheExpiry = c
-	}
-}
-
 // WithNetwork is a CallOption which sets the network attribute
 func WithNetwork(n string) CallOption {
 	return func(o *CallOptions) {

client/service/proto/client.pb.go

@@ -1,402 +0,0 @@
-// Code generated by protoc-gen-go. DO NOT EDIT.
-// source: client/service/proto/client.proto
-
-package go_micro_client
-
-import (
-	context "context"
-	fmt "fmt"
-	proto "github.com/golang/protobuf/proto"
-	grpc "google.golang.org/grpc"
-	codes "google.golang.org/grpc/codes"
-	status "google.golang.org/grpc/status"
-	math "math"
-)
-
-// Reference imports to suppress errors if they are not otherwise used.
-var _ = proto.Marshal
-var _ = fmt.Errorf
-var _ = math.Inf
-
-// This is a compile-time assertion to ensure that this generated file
-// is compatible with the proto package it is being compiled against.
-// A compilation error at this line likely means your copy of the
-// proto package needs to be updated.
-const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package
-
-type Request struct {
-	Service              string   `protobuf:"bytes,1,opt,name=service,proto3" json:"service,omitempty"`
-	Endpoint             string   `protobuf:"bytes,2,opt,name=endpoint,proto3" json:"endpoint,omitempty"`
-	ContentType          string   `protobuf:"bytes,3,opt,name=content_type,json=contentType,proto3" json:"content_type,omitempty"`
-	Body                 []byte   `protobuf:"bytes,4,opt,name=body,proto3" json:"body,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
-}
-
-func (m *Request) Reset()         { *m = Request{} }
-func (m *Request) String() string { return proto.CompactTextString(m) }
-func (*Request) ProtoMessage()    {}
-func (*Request) Descriptor() ([]byte, []int) {
-	return fileDescriptor_27c3d425ddd1a066, []int{0}
-}
-
-func (m *Request) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_Request.Unmarshal(m, b)
-}
-func (m *Request) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_Request.Marshal(b, m, deterministic)
-}
-func (m *Request) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_Request.Merge(m, src)
-}
-func (m *Request) XXX_Size() int {
-	return xxx_messageInfo_Request.Size(m)
-}
-func (m *Request) XXX_DiscardUnknown() {
-	xxx_messageInfo_Request.DiscardUnknown(m)
-}
-
-var xxx_messageInfo_Request proto.InternalMessageInfo
-
-func (m *Request) GetService() string {
-	if m != nil {
-		return m.Service
-	}
-	return ""
-}
-
-func (m *Request) GetEndpoint() string {
-	if m != nil {
-		return m.Endpoint
-	}
-	return ""
-}
-
-func (m *Request) GetContentType() string {
-	if m != nil {
-		return m.ContentType
-	}
-	return ""
-}
-
-func (m *Request) GetBody() []byte {
-	if m != nil {
-		return m.Body
-	}
-	return nil
-}
-
-type Response struct {
-	Body                 []byte   `protobuf:"bytes,1,opt,name=body,proto3" json:"body,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
-}
-
-func (m *Response) Reset()         { *m = Response{} }
-func (m *Response) String() string { return proto.CompactTextString(m) }
-func (*Response) ProtoMessage()    {}
-func (*Response) Descriptor() ([]byte, []int) {
-	return fileDescriptor_27c3d425ddd1a066, []int{1}
-}
-
-func (m *Response) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_Response.Unmarshal(m, b)
-}
-func (m *Response) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_Response.Marshal(b, m, deterministic)
-}
-func (m *Response) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_Response.Merge(m, src)
-}
-func (m *Response) XXX_Size() int {
-	return xxx_messageInfo_Response.Size(m)
-}
-func (m *Response) XXX_DiscardUnknown() {
-	xxx_messageInfo_Response.DiscardUnknown(m)
-}
-
-var xxx_messageInfo_Response proto.InternalMessageInfo
-
-func (m *Response) GetBody() []byte {
-	if m != nil {
-		return m.Body
-	}
-	return nil
-}
-
-type Message struct {
-	Topic                string   `protobuf:"bytes,1,opt,name=topic,proto3" json:"topic,omitempty"`
-	ContentType          string   `protobuf:"bytes,2,opt,name=content_type,json=contentType,proto3" json:"content_type,omitempty"`
-	Body                 []byte   `protobuf:"bytes,3,opt,name=body,proto3" json:"body,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
-}
-
-func (m *Message) Reset()         { *m = Message{} }
-func (m *Message) String() string { return proto.CompactTextString(m) }
-func (*Message) ProtoMessage()    {}
-func (*Message) Descriptor() ([]byte, []int) {
-	return fileDescriptor_27c3d425ddd1a066, []int{2}
-}
-
-func (m *Message) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_Message.Unmarshal(m, b)
-}
-func (m *Message) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_Message.Marshal(b, m, deterministic)
-}
-func (m *Message) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_Message.Merge(m, src)
-}
-func (m *Message) XXX_Size() int {
-	return xxx_messageInfo_Message.Size(m)
-}
-func (m *Message) XXX_DiscardUnknown() {
-	xxx_messageInfo_Message.DiscardUnknown(m)
-}
-
-var xxx_messageInfo_Message proto.InternalMessageInfo
-
-func (m *Message) GetTopic() string {
-	if m != nil {
-		return m.Topic
-	}
-	return ""
-}
-
-func (m *Message) GetContentType() string {
-	if m != nil {
-		return m.ContentType
-	}
-	return ""
-}
-
-func (m *Message) GetBody() []byte {
-	if m != nil {
-		return m.Body
-	}
-	return nil
-}
-
-func init() {
-	proto.RegisterType((*Request)(nil), "go.micro.client.Request")
-	proto.RegisterType((*Response)(nil), "go.micro.client.Response")
-	proto.RegisterType((*Message)(nil), "go.micro.client.Message")
-}
-
-func init() { proto.RegisterFile("client/service/proto/client.proto", fileDescriptor_27c3d425ddd1a066) }
-
-var fileDescriptor_27c3d425ddd1a066 = []byte{
-	// 267 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x91, 0xc1, 0x4b, 0xc3, 0x30,
-	0x14, 0xc6, 0x97, 0x6d, 0xb6, 0xf3, 0x39, 0x10, 0x1e, 0x1e, 0x62, 0x0f, 0xb2, 0xf5, 0xd4, 0x53,
-	0x2b, 0x7a, 0x16, 0x0f, 0x3d, 0x0b, 0x52, 0xc5, 0xab, 0xb4, 0xd9, 0x63, 0x06, 0xba, 0x24, 0x36,
-	0xd9, 0xa0, 0x7f, 0xa4, 0xff, 0x93, 0x90, 0x46, 0x27, 0xba, 0x5d, 0xbc, 0xe5, 0xfb, 0x7e, 0xe4,
-	0x7b, 0x2f, 0x5f, 0x60, 0x29, 0x5a, 0x49, 0xca, 0x15, 0x96, 0xba, 0x9d, 0x14, 0x54, 0x98, 0x4e,
-	0x3b, 0x5d, 0x0c, 0x66, 0xee, 0x05, 0x9e, 0xaf, 0x75, 0xbe, 0x91, 0xa2, 0xd3, 0xf9, 0x60, 0xa7,
-	0x3b, 0x88, 0x2b, 0x7a, 0xdf, 0x92, 0x75, 0xc8, 0x21, 0x0e, 0x37, 0x39, 0x5b, 0xb0, 0xec, 0xb4,
-	0xfa, 0x92, 0x98, 0xc0, 0x8c, 0xd4, 0xca, 0x68, 0xa9, 0x1c, 0x1f, 0x7b, 0xf4, 0xad, 0x71, 0x09,
-	0x73, 0xa1, 0x95, 0x23, 0xe5, 0x5e, 0x5d, 0x6f, 0x88, 0x4f, 0x3c, 0x3f, 0x0b, 0xde, 0x73, 0x6f,
-	0x08, 0x11, 0xa6, 0x8d, 0x5e, 0xf5, 0x7c, 0xba, 0x60, 0xd9, 0xbc, 0xf2, 0xe7, 0xf4, 0x0a, 0x66,
-	0x15, 0x59, 0xa3, 0x95, 0xdd, 0x73, 0xf6, 0x83, 0xbf, 0x40, 0xfc, 0x40, 0xd6, 0xd6, 0x6b, 0xc2,
-	0x0b, 0x38, 0x71, 0xda, 0x48, 0x11, 0xb6, 0x1a, 0xc4, 0x9f, 0xb9, 0xe3, 0xe3, 0x73, 0x27, 0xfb,
-	0xdc, 0x9b, 0x0f, 0x06, 0x51, 0xe9, 0x9f, 0x8e, 0x77, 0x30, 0x2d, 0xeb, 0xb6, 0x45, 0x9e, 0xff,
-	0x2a, 0x25, 0x0f, 0x8d, 0x24, 0x97, 0x07, 0xc8, 0xb0, 0x73, 0x3a, 0xc2, 0x12, 0xa2, 0x27, 0xd7,
-	0x51, 0xbd, 0xf9, 0x67, 0x40, 0xc6, 0xae, 0x19, 0xde, 0x43, 0xfc, 0xb8, 0x6d, 0x5a, 0x69, 0xdf,
-	0x0e, 0xa4, 0x84, 0x02, 0x92, 0xa3, 0x24, 0x1d, 0x35, 0x91, 0xff, 0xd7, 0xdb, 0xcf, 0x00, 0x00,
-	0x00, 0xff, 0xff, 0xd6, 0x3f, 0xc3, 0xa1, 0xfc, 0x01, 0x00, 0x00,
-}
-
-// Reference imports to suppress errors if they are not otherwise used.
-var _ context.Context
-var _ grpc.ClientConn
-
-// This is a compile-time assertion to ensure that this generated file
-// is compatible with the grpc package it is being compiled against.
-const _ = grpc.SupportPackageIsVersion4
-
-// ClientClient is the client API for Client service.
-//
-// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream.
-type ClientClient interface {
-	// Call allows a single request to be made
-	Call(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error)
-	// Stream is a bidirectional stream
-	Stream(ctx context.Context, opts ...grpc.CallOption) (Client_StreamClient, error)
-	// Publish publishes a message and returns an empty Message
-	Publish(ctx context.Context, in *Message, opts ...grpc.CallOption) (*Message, error)
-}
-
-type clientClient struct {
-	cc *grpc.ClientConn
-}
-
-func NewClientClient(cc *grpc.ClientConn) ClientClient {
-	return &clientClient{cc}
-}
-
-func (c *clientClient) Call(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) {
-	out := new(Response)
-	err := c.cc.Invoke(ctx, "/go.micro.client.Client/Call", in, out, opts...)
-	if err != nil {
-		return nil, err
-	}
-	return out, nil
-}
-
-func (c *clientClient) Stream(ctx context.Context, opts ...grpc.CallOption) (Client_StreamClient, error) {
-	stream, err := c.cc.NewStream(ctx, &_Client_serviceDesc.Streams[0], "/go.micro.client.Client/Stream", opts...)
-	if err != nil {
-		return nil, err
-	}
-	x := &clientStreamClient{stream}
-	return x, nil
-}
-
-type Client_StreamClient interface {
-	Send(*Request) error
-	Recv() (*Response, error)
-	grpc.ClientStream
-}
-
-type clientStreamClient struct {
-	grpc.ClientStream
-}
-
-func (x *clientStreamClient) Send(m *Request) error {
-	return x.ClientStream.SendMsg(m)
-}
-
-func (x *clientStreamClient) Recv() (*Response, error) {
-	m := new(Response)
-	if err := x.ClientStream.RecvMsg(m); err != nil {
-		return nil, err
-	}
-	return m, nil
-}
-
-func (c *clientClient) Publish(ctx context.Context, in *Message, opts ...grpc.CallOption) (*Message, error) {
-	out := new(Message)
-	err := c.cc.Invoke(ctx, "/go.micro.client.Client/Publish", in, out, opts...)
-	if err != nil {
-		return nil, err
-	}
-	return out, nil
-}
-
-// ClientServer is the server API for Client service.
-type ClientServer interface {
-	// Call allows a single request to be made
-	Call(context.Context, *Request) (*Response, error)
-	// Stream is a bidirectional stream
-	Stream(Client_StreamServer) error
-	// Publish publishes a message and returns an empty Message
-	Publish(context.Context, *Message) (*Message, error)
-}
-
-// UnimplementedClientServer can be embedded to have forward compatible implementations.
-type UnimplementedClientServer struct {
-}
-
-func (*UnimplementedClientServer) Call(ctx context.Context, req *Request) (*Response, error) {
-	return nil, status.Errorf(codes.Unimplemented, "method Call not implemented")
-}
-func (*UnimplementedClientServer) Stream(srv Client_StreamServer) error {
-	return status.Errorf(codes.Unimplemented, "method Stream not implemented")
-}
-func (*UnimplementedClientServer) Publish(ctx context.Context, req *Message) (*Message, error) {
-	return nil, status.Errorf(codes.Unimplemented, "method Publish not implemented")
-}
-
-func RegisterClientServer(s *grpc.Server, srv ClientServer) {
-	s.RegisterService(&_Client_serviceDesc, srv)
-}
-
-func _Client_Call_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
-	in := new(Request)
-	if err := dec(in); err != nil {
-		return nil, err
-	}
-	if interceptor == nil {
-		return srv.(ClientServer).Call(ctx, in)
-	}
-	info := &grpc.UnaryServerInfo{
-		Server:     srv,
-		FullMethod: "/go.micro.client.Client/Call",
-	}
-	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
-		return srv.(ClientServer).Call(ctx, req.(*Request))
-	}
-	return interceptor(ctx, in, info, handler)
-}
-
-func _Client_Stream_Handler(srv interface{}, stream grpc.ServerStream) error {
-	return srv.(ClientServer).Stream(&clientStreamServer{stream})
-}
-
-type Client_StreamServer interface {
-	Send(*Response) error
-	Recv() (*Request, error)
-	grpc.ServerStream
-}
-
-type clientStreamServer struct {
-	grpc.ServerStream
-}
-
-func (x *clientStreamServer) Send(m *Response) error {
-	return x.ServerStream.SendMsg(m)
-}
-
-func (x *clientStreamServer) Recv() (*Request, error) {
-	m := new(Request)
-	if err := x.ServerStream.RecvMsg(m); err != nil {
-		return nil, err
-	}
-	return m, nil
-}
-
-func _Client_Publish_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
-	in := new(Message)
-	if err := dec(in); err != nil {
-		return nil, err
-	}
-	if interceptor == nil {
-		return srv.(ClientServer).Publish(ctx, in)
-	}
-	info := &grpc.UnaryServerInfo{
-		Server:     srv,
-		FullMethod: "/go.micro.client.Client/Publish",
-	}
-	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
-		return srv.(ClientServer).Publish(ctx, req.(*Message))
-	}
-	return interceptor(ctx, in, info, handler)
-}
-
-var _Client_serviceDesc = grpc.ServiceDesc{
-	ServiceName: "go.micro.client.Client",
-	HandlerType: (*ClientServer)(nil),
-	Methods: []grpc.MethodDesc{
-		{
-			MethodName: "Call",
-			Handler:    _Client_Call_Handler,
-		},
-		{
-			MethodName: "Publish",
-			Handler:    _Client_Publish_Handler,
-		},
-	},
-	Streams: []grpc.StreamDesc{
-		{
-			StreamName:    "Stream",
-			Handler:       _Client_Stream_Handler,
-			ServerStreams: true,
-			ClientStreams: true,
-		},
-	},
-	Metadata: "client/service/proto/client.proto",
-}

client/service/proto/client.pb.micro.go

@@ -1,215 +0,0 @@
-// Code generated by protoc-gen-micro. DO NOT EDIT.
-// source: client/service/proto/client.proto
-
-package go_micro_client
-
-import (
-	fmt "fmt"
-	proto "github.com/golang/protobuf/proto"
-	math "math"
-)
-
-import (
-	context "context"
-	api "github.com/micro/go-micro/v3/api"
-	client "github.com/micro/go-micro/v3/client"
-	server "github.com/micro/go-micro/v3/server"
-)
-
-// Reference imports to suppress errors if they are not otherwise used.
-var _ = proto.Marshal
-var _ = fmt.Errorf
-var _ = math.Inf
-
-// This is a compile-time assertion to ensure that this generated file
-// is compatible with the proto package it is being compiled against.
-// A compilation error at this line likely means your copy of the
-// proto package needs to be updated.
-const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package
-
-// Reference imports to suppress errors if they are not otherwise used.
-var _ api.Endpoint
-var _ context.Context
-var _ client.Option
-var _ server.Option
-
-// Api Endpoints for Client service
-
-func NewClientEndpoints() []*api.Endpoint {
-	return []*api.Endpoint{}
-}
-
-// Client API for Client service
-
-type ClientService interface {
-	// Call allows a single request to be made
-	Call(ctx context.Context, in *Request, opts ...client.CallOption) (*Response, error)
-	// Stream is a bidirectional stream
-	Stream(ctx context.Context, opts ...client.CallOption) (Client_StreamService, error)
-	// Publish publishes a message and returns an empty Message
-	Publish(ctx context.Context, in *Message, opts ...client.CallOption) (*Message, error)
-}
-
-type clientService struct {
-	c    client.Client
-	name string
-}
-
-func NewClientService(name string, c client.Client) ClientService {
-	return &clientService{
-		c:    c,
-		name: name,
-	}
-}
-
-func (c *clientService) Call(ctx context.Context, in *Request, opts ...client.CallOption) (*Response, error) {
-	req := c.c.NewRequest(c.name, "Client.Call", in)
-	out := new(Response)
-	err := c.c.Call(ctx, req, out, opts...)
-	if err != nil {
-		return nil, err
-	}
-	return out, nil
-}
-
-func (c *clientService) Stream(ctx context.Context, opts ...client.CallOption) (Client_StreamService, error) {
-	req := c.c.NewRequest(c.name, "Client.Stream", &Request{})
-	stream, err := c.c.Stream(ctx, req, opts...)
-	if err != nil {
-		return nil, err
-	}
-	return &clientServiceStream{stream}, nil
-}
-
-type Client_StreamService interface {
-	Context() context.Context
-	SendMsg(interface{}) error
-	RecvMsg(interface{}) error
-	Close() error
-	Send(*Request) error
-	Recv() (*Response, error)
-}
-
-type clientServiceStream struct {
-	stream client.Stream
-}
-
-func (x *clientServiceStream) Close() error {
-	return x.stream.Close()
-}
-
-func (x *clientServiceStream) Context() context.Context {
-	return x.stream.Context()
-}
-
-func (x *clientServiceStream) SendMsg(m interface{}) error {
-	return x.stream.Send(m)
-}
-
-func (x *clientServiceStream) RecvMsg(m interface{}) error {
-	return x.stream.Recv(m)
-}
-
-func (x *clientServiceStream) Send(m *Request) error {
-	return x.stream.Send(m)
-}
-
-func (x *clientServiceStream) Recv() (*Response, error) {
-	m := new(Response)
-	err := x.stream.Recv(m)
-	if err != nil {
-		return nil, err
-	}
-	return m, nil
-}
-
-func (c *clientService) Publish(ctx context.Context, in *Message, opts ...client.CallOption) (*Message, error) {
-	req := c.c.NewRequest(c.name, "Client.Publish", in)
-	out := new(Message)
-	err := c.c.Call(ctx, req, out, opts...)
-	if err != nil {
-		return nil, err
-	}
-	return out, nil
-}
-
-// Server API for Client service
-
-type ClientHandler interface {
-	// Call allows a single request to be made
-	Call(context.Context, *Request, *Response) error
-	// Stream is a bidirectional stream
-	Stream(context.Context, Client_StreamStream) error
-	// Publish publishes a message and returns an empty Message
-	Publish(context.Context, *Message, *Message) error
-}
-
-func RegisterClientHandler(s server.Server, hdlr ClientHandler, opts ...server.HandlerOption) error {
-	type client interface {
-		Call(ctx context.Context, in *Request, out *Response) error
-		Stream(ctx context.Context, stream server.Stream) error
-		Publish(ctx context.Context, in *Message, out *Message) error
-	}
-	type Client struct {
-		client
-	}
-	h := &clientHandler{hdlr}
-	return s.Handle(s.NewHandler(&Client{h}, opts...))
-}
-
-type clientHandler struct {
-	ClientHandler
-}
-
-func (h *clientHandler) Call(ctx context.Context, in *Request, out *Response) error {
-	return h.ClientHandler.Call(ctx, in, out)
-}
-
-func (h *clientHandler) Stream(ctx context.Context, stream server.Stream) error {
-	return h.ClientHandler.Stream(ctx, &clientStreamStream{stream})
-}
-
-type Client_StreamStream interface {
-	Context() context.Context
-	SendMsg(interface{}) error
-	RecvMsg(interface{}) error
-	Close() error
-	Send(*Response) error
-	Recv() (*Request, error)
-}
-
-type clientStreamStream struct {
-	stream server.Stream
-}
-
-func (x *clientStreamStream) Close() error {
-	return x.stream.Close()
-}
-
-func (x *clientStreamStream) Context() context.Context {
-	return x.stream.Context()
-}
-
-func (x *clientStreamStream) SendMsg(m interface{}) error {
-	return x.stream.Send(m)
-}
-
-func (x *clientStreamStream) RecvMsg(m interface{}) error {
-	return x.stream.Recv(m)
-}
-
-func (x *clientStreamStream) Send(m *Response) error {
-	return x.stream.Send(m)
-}
-
-func (x *clientStreamStream) Recv() (*Request, error) {
-	m := new(Request)
-	if err := x.stream.Recv(m); err != nil {
-		return nil, err
-	}
-	return m, nil
-}
-
-func (h *clientHandler) Publish(ctx context.Context, in *Message, out *Message) error {
-	return h.ClientHandler.Publish(ctx, in, out)
-}

client/service/proto/client.proto

@@ -1,30 +0,0 @@
-syntax = "proto3";
-
-package go.micro.client;
-
-// Client is the micro client interface
-service Client {
-	// Call allows a single request to be made
-	rpc Call(Request) returns (Response) {};
-	// Stream is a bidirectional stream
-	rpc Stream(stream Request) returns (stream Response) {};
-	// Publish publishes a message and returns an empty Message
-	rpc Publish(Message) returns (Message) {};
-}
-
-message Request {
-	string service = 1;
-	string endpoint = 2;
-	string content_type = 3;
-	bytes body = 4;
-}
-
-message Response {
-	bytes body = 1;
-}
-
-message Message {
-	string topic = 1;
-	string content_type = 2;
-	bytes body = 3;
-}

client/test_request.go

@@ -1,65 +0,0 @@
-package client
-
-import (
-	"github.com/micro/go-micro/v3/codec"
-)
-
-type testRequest struct {
-	service     string
-	method      string
-	endpoint    string
-	contentType string
-	codec       codec.Codec
-	body        interface{}
-	opts        RequestOptions
-}
-
-func newRequest(service, endpoint string, request interface{}, contentType string, reqOpts ...RequestOption) Request {
-	var opts RequestOptions
-
-	for _, o := range reqOpts {
-		o(&opts)
-	}
-
-	// set the content-type specified
-	if len(opts.ContentType) > 0 {
-		contentType = opts.ContentType
-	}
-
-	return &testRequest{
-		service:     service,
-		method:      endpoint,
-		endpoint:    endpoint,
-		body:        request,
-		contentType: contentType,
-		opts:        opts,
-	}
-}
-
-func (r *testRequest) ContentType() string {
-	return r.contentType
-}
-
-func (r *testRequest) Service() string {
-	return r.service
-}
-
-func (r *testRequest) Method() string {
-	return r.method
-}
-
-func (r *testRequest) Endpoint() string {
-	return r.endpoint
-}
-
-func (r *testRequest) Body() interface{} {
-	return r.body
-}
-
-func (r *testRequest) Codec() codec.Writer {
-	return r.codec
-}
-
-func (r *testRequest) Stream() bool {
-	return r.opts.Stream
-}

config/config.go

@@ -2,49 +2,42 @@
 package config
 
 import (
-	"context"
+	"time"
-
-	"github.com/micro/go-micro/v3/config/loader"
-	"github.com/micro/go-micro/v3/config/reader"
-	"github.com/micro/go-micro/v3/config/source"
 )
 
 // Config is an interface abstraction for dynamic configuration
 type Config interface {
-	// provide the reader.Values interface
+	Get(path string, options ...Option) (Value, error)
-	reader.Values
+	Set(path string, val interface{}, options ...Option) error
-	// Init the config
+	Delete(path string, options ...Option) error
-	Init(opts ...Option) error
-	// Options in the config
-	Options() Options
-	// Stop the config loader/watcher
-	Close() error
-	// Load config sources
-	Load(source ...source.Source) error
-	// Force a source changeset sync
-	Sync() error
-	// Watch a value for changes
-	Watch(path ...string) (Watcher, error)
 }
 
-// Watcher is the config watcher
+// Value represents a value of any type
-type Watcher interface {
+type Value interface {
-	Next() (reader.Value, error)
+	Exists() bool
-	Stop() error
+	Bool(def bool) bool
+	Int(def int) int
+	String(def string) string
+	Float64(def float64) float64
+	Duration(def time.Duration) time.Duration
+	StringSlice(def []string) []string
+	StringMap(def map[string]string) map[string]string
+	Scan(val interface{}) error
+	Bytes() []byte
 }
 
 type Options struct {
-	Loader loader.Loader
+	Secret bool
-	Reader reader.Reader
-	Source []source.Source
-
-	// for alternative data
-	Context context.Context
 }
 
 type Option func(o *Options)
 
-// NewConfig returns new config
+func Secret(b bool) Option {
-func NewConfig(opts ...Option) (Config, error) {
+	return func(o *Options) {
-	return newConfig(opts...)
+		o.Secret = b
+	}
+}
+
+type Secrets interface {
+	Config
 }

config/default.go

@@ -1,301 +0,0 @@
-package config
-
-import (
-	"bytes"
-	"sync"
-	"time"
-
-	"github.com/micro/go-micro/v3/config/loader"
-	"github.com/micro/go-micro/v3/config/loader/memory"
-	"github.com/micro/go-micro/v3/config/reader"
-	"github.com/micro/go-micro/v3/config/reader/json"
-	"github.com/micro/go-micro/v3/config/source"
-)
-
-type config struct {
-	exit chan bool
-	opts Options
-
-	sync.RWMutex
-	// the current snapshot
-	snap *loader.Snapshot
-	// the current values
-	vals reader.Values
-}
-
-type watcher struct {
-	lw    loader.Watcher
-	rd    reader.Reader
-	path  []string
-	value reader.Value
-}
-
-func newConfig(opts ...Option) (Config, error) {
-	var c config
-
-	if err := c.Init(opts...); err != nil {
-		return nil, err
-	}
-
-	go c.run()
-	return &c, nil
-}
-
-func (c *config) Init(opts ...Option) error {
-	c.opts = Options{
-		Reader: json.NewReader(),
-	}
-	c.exit = make(chan bool)
-	for _, o := range opts {
-		o(&c.opts)
-	}
-
-	// default loader uses the configured reader
-	if c.opts.Loader == nil {
-		c.opts.Loader = memory.NewLoader(memory.WithReader(c.opts.Reader))
-	}
-
-	err := c.opts.Loader.Load(c.opts.Source...)
-	if err != nil {
-		return err
-	}
-
-	c.snap, err = c.opts.Loader.Snapshot()
-	if err != nil {
-		return err
-	}
-
-	c.vals, err = c.opts.Reader.Values(c.snap.ChangeSet)
-	if err != nil {
-		return err
-	}
-
-	return nil
-}
-
-func (c *config) Options() Options {
-	return c.opts
-}
-
-func (c *config) run() {
-	watch := func(w loader.Watcher) error {
-		for {
-			// get changeset
-			snap, err := w.Next()
-			if err != nil {
-				return err
-			}
-
-			c.Lock()
-
-			if c.snap != nil && c.snap.Version >= snap.Version {
-				c.Unlock()
-				continue
-			}
-
-			// save
-			c.snap = snap
-
-			// set values
-			c.vals, _ = c.opts.Reader.Values(snap.ChangeSet)
-
-			c.Unlock()
-		}
-	}
-
-	for {
-		w, err := c.opts.Loader.Watch()
-		if err != nil {
-			time.Sleep(time.Second)
-			continue
-		}
-
-		done := make(chan bool)
-
-		// the stop watch func
-		go func() {
-			select {
-			case <-done:
-			case <-c.exit:
-			}
-			w.Stop()
-		}()
-
-		// block watch
-		if err := watch(w); err != nil {
-			// do something better
-			time.Sleep(time.Second)
-		}
-
-		// close done chan
-		close(done)
-
-		// if the config is closed exit
-		select {
-		case <-c.exit:
-			return
-		default:
-		}
-	}
-}
-
-func (c *config) Map() map[string]interface{} {
-	c.RLock()
-	defer c.RUnlock()
-	return c.vals.Map()
-}
-
-func (c *config) Scan(v interface{}) error {
-	c.RLock()
-	defer c.RUnlock()
-	return c.vals.Scan(v)
-}
-
-// sync loads all the sources, calls the parser and updates the config
-func (c *config) Sync() error {
-	if err := c.opts.Loader.Sync(); err != nil {
-		return err
-	}
-
-	snap, err := c.opts.Loader.Snapshot()
-	if err != nil {
-		return err
-	}
-
-	c.Lock()
-	defer c.Unlock()
-
-	c.snap = snap
-	vals, err := c.opts.Reader.Values(snap.ChangeSet)
-	if err != nil {
-		return err
-	}
-	c.vals = vals
-
-	return nil
-}
-
-func (c *config) Close() error {
-	select {
-	case <-c.exit:
-		return nil
-	default:
-		close(c.exit)
-	}
-	return nil
-}
-
-func (c *config) Get(path ...string) reader.Value {
-	c.RLock()
-	defer c.RUnlock()
-
-	// did sync actually work?
-	if c.vals != nil {
-		return c.vals.Get(path...)
-	}
-
-	// no value
-	return newValue()
-}
-
-func (c *config) Set(val interface{}, path ...string) {
-	c.Lock()
-	defer c.Unlock()
-
-	if c.vals != nil {
-		c.vals.Set(val, path...)
-	}
-
-	return
-}
-
-func (c *config) Del(path ...string) {
-	c.Lock()
-	defer c.Unlock()
-
-	if c.vals != nil {
-		c.vals.Del(path...)
-	}
-
-	return
-}
-
-func (c *config) Bytes() []byte {
-	c.RLock()
-	defer c.RUnlock()
-
-	if c.vals == nil {
-		return []byte{}
-	}
-
-	return c.vals.Bytes()
-}
-
-func (c *config) Load(sources ...source.Source) error {
-	if err := c.opts.Loader.Load(sources...); err != nil {
-		return err
-	}
-
-	snap, err := c.opts.Loader.Snapshot()
-	if err != nil {
-		return err
-	}
-
-	c.Lock()
-	defer c.Unlock()
-
-	c.snap = snap
-	vals, err := c.opts.Reader.Values(snap.ChangeSet)
-	if err != nil {
-		return err
-	}
-	c.vals = vals
-
-	return nil
-}
-
-func (c *config) Watch(path ...string) (Watcher, error) {
-	value := c.Get(path...)
-
-	w, err := c.opts.Loader.Watch(path...)
-	if err != nil {
-		return nil, err
-	}
-
-	return &watcher{
-		lw:    w,
-		rd:    c.opts.Reader,
-		path:  path,
-		value: value,
-	}, nil
-}
-
-func (c *config) String() string {
-	return "config"
-}
-
-func (w *watcher) Next() (reader.Value, error) {
-	for {
-		s, err := w.lw.Next()
-		if err != nil {
-			return nil, err
-		}
-
-		// only process changes
-		if bytes.Equal(w.value.Bytes(), s.ChangeSet.Data) {
-			continue
-		}
-
-		v, err := w.rd.Values(s.ChangeSet)
-		if err != nil {
-			return nil, err
-		}
-
-		w.value = v.Get()
-		return w.value, nil
-	}
-}
-
-func (w *watcher) Stop() error {
-	return w.lw.Stop()
-}

config/default_test.go

@@ -1,166 +0,0 @@
-package config
-
-import (
-	"fmt"
-	"os"
-	"path/filepath"
-	"runtime"
-	"strings"
-	"testing"
-	"time"
-
-	"github.com/micro/go-micro/v3/config/source"
-	"github.com/micro/go-micro/v3/config/source/env"
-	"github.com/micro/go-micro/v3/config/source/file"
-	"github.com/micro/go-micro/v3/config/source/memory"
-)
-
-func createFileForIssue18(t *testing.T, content string) *os.File {
-	data := []byte(content)
-	path := filepath.Join(os.TempDir(), fmt.Sprintf("file.%d", time.Now().UnixNano()))
-	fh, err := os.Create(path)
-	if err != nil {
-		t.Error(err)
-	}
-	_, err = fh.Write(data)
-	if err != nil {
-		t.Error(err)
-	}
-
-	return fh
-}
-
-func createFileForTest(t *testing.T) *os.File {
-	data := []byte(`{"foo": "bar"}`)
-	path := filepath.Join(os.TempDir(), fmt.Sprintf("file.%d", time.Now().UnixNano()))
-	fh, err := os.Create(path)
-	if err != nil {
-		t.Error(err)
-	}
-	_, err = fh.Write(data)
-	if err != nil {
-		t.Error(err)
-	}
-
-	return fh
-}
-
-func TestConfigLoadWithGoodFile(t *testing.T) {
-	fh := createFileForTest(t)
-	path := fh.Name()
-	defer func() {
-		fh.Close()
-		os.Remove(path)
-	}()
-
-	// Create new config
-	conf, err := NewConfig()
-	if err != nil {
-		t.Fatalf("Expected no error but got %v", err)
-	}
-	// Load file source
-	if err := conf.Load(file.NewSource(
-		file.WithPath(path),
-	)); err != nil {
-		t.Fatalf("Expected no error but got %v", err)
-	}
-}
-
-func TestConfigLoadWithInvalidFile(t *testing.T) {
-	fh := createFileForTest(t)
-	path := fh.Name()
-	defer func() {
-		fh.Close()
-		os.Remove(path)
-	}()
-
-	// Create new config
-	conf, err := NewConfig()
-	if err != nil {
-		t.Fatalf("Expected no error but got %v", err)
-	}
-	// Load file source
-	err = conf.Load(file.NewSource(
-		file.WithPath(path),
-		file.WithPath("/i/do/not/exists.json"),
-	))
-
-	if err == nil {
-		t.Fatal("Expected error but none !")
-	}
-	if !strings.Contains(fmt.Sprintf("%v", err), "/i/do/not/exists.json") {
-		t.Fatalf("Expected error to contain the unexisting file but got %v", err)
-	}
-}
-
-func TestConfigMerge(t *testing.T) {
-	fh := createFileForIssue18(t, `{
-  "amqp": {
-    "host": "rabbit.platform",
-    "port": 80
-  },
-  "handler": {
-    "exchange": "springCloudBus"
-  }
-}`)
-	path := fh.Name()
-	defer func() {
-		fh.Close()
-		os.Remove(path)
-	}()
-	os.Setenv("AMQP_HOST", "rabbit.testing.com")
-
-	conf, err := NewConfig()
-	if err != nil {
-		t.Fatalf("Expected no error but got %v", err)
-	}
-	if err := conf.Load(
-		file.NewSource(
-			file.WithPath(path),
-		),
-		env.NewSource(),
-	); err != nil {
-		t.Fatalf("Expected no error but got %v", err)
-	}
-
-	actualHost := conf.Get("amqp", "host").String("backup")
-	if actualHost != "rabbit.testing.com" {
-		t.Fatalf("Expected %v but got %v",
-			"rabbit.testing.com",
-			actualHost)
-	}
-}
-
-func equalS(t *testing.T, actual, expect string) {
-	if actual != expect {
-		t.Errorf("Expected %s but got %s", actual, expect)
-	}
-}
-
-func TestConfigWatcherDirtyOverrite(t *testing.T) {
-	n := runtime.GOMAXPROCS(0)
-	defer runtime.GOMAXPROCS(n)
-
-	runtime.GOMAXPROCS(1)
-
-	l := 100
-
-	ss := make([]source.Source, l, l)
-
-	for i := 0; i < l; i++ {
-		ss[i] = memory.NewSource(memory.WithJSON([]byte(fmt.Sprintf(`{"key%d": "val%d"}`, i, i))))
-	}
-
-	conf, _ := NewConfig()
-
-	for _, s := range ss {
-		_ = conf.Load(s)
-	}
-	runtime.Gosched()
-
-	for i, _ := range ss {
-		k := fmt.Sprintf("key%d", i)
-		v := fmt.Sprintf("val%d", i)
-		equalS(t, conf.Get(k).String(""), v)
-	}
-}

config/encoder/encoder.go

@@ -1,8 +0,0 @@
-// Package encoder handles source encoding formats
-package encoder
-
-type Encoder interface {
-	Encode(interface{}) ([]byte, error)
-	Decode([]byte, interface{}) error
-	String() string
-}

config/encoder/hcl/hcl.go

@@ -1,26 +0,0 @@
-package hcl
-
-import (
-	"encoding/json"
-
-	"github.com/hashicorp/hcl"
-	"github.com/micro/go-micro/v3/config/encoder"
-)
-
-type hclEncoder struct{}
-
-func (h hclEncoder) Encode(v interface{}) ([]byte, error) {
-	return json.Marshal(v)
-}
-
-func (h hclEncoder) Decode(d []byte, v interface{}) error {
-	return hcl.Unmarshal(d, v)
-}
-
-func (h hclEncoder) String() string {
-	return "hcl"
-}
-
-func NewEncoder() encoder.Encoder {
-	return hclEncoder{}
-}

config/encoder/json/json.go

@@ -1,25 +0,0 @@
-package json
-
-import (
-	"encoding/json"
-
-	"github.com/micro/go-micro/v3/config/encoder"
-)
-
-type jsonEncoder struct{}
-
-func (j jsonEncoder) Encode(v interface{}) ([]byte, error) {
-	return json.Marshal(v)
-}
-
-func (j jsonEncoder) Decode(d []byte, v interface{}) error {
-	return json.Unmarshal(d, v)
-}
-
-func (j jsonEncoder) String() string {
-	return "json"
-}
-
-func NewEncoder() encoder.Encoder {
-	return jsonEncoder{}
-}

config/encoder/toml/toml.go

@@ -1,32 +0,0 @@
-package toml
-
-import (
-	"bytes"
-
-	"github.com/BurntSushi/toml"
-	"github.com/micro/go-micro/v3/config/encoder"
-)
-
-type tomlEncoder struct{}
-
-func (t tomlEncoder) Encode(v interface{}) ([]byte, error) {
-	b := bytes.NewBuffer(nil)
-	defer b.Reset()
-	err := toml.NewEncoder(b).Encode(v)
-	if err != nil {
-		return nil, err
-	}
-	return b.Bytes(), nil
-}
-
-func (t tomlEncoder) Decode(d []byte, v interface{}) error {
-	return toml.Unmarshal(d, v)
-}
-
-func (t tomlEncoder) String() string {
-	return "toml"
-}
-
-func NewEncoder() encoder.Encoder {
-	return tomlEncoder{}
-}

config/encoder/xml/xml.go

@@ -1,25 +0,0 @@
-package xml
-
-import (
-	"encoding/xml"
-
-	"github.com/micro/go-micro/v3/config/encoder"
-)
-
-type xmlEncoder struct{}
-
-func (x xmlEncoder) Encode(v interface{}) ([]byte, error) {
-	return xml.Marshal(v)
-}
-
-func (x xmlEncoder) Decode(d []byte, v interface{}) error {
-	return xml.Unmarshal(d, v)
-}
-
-func (x xmlEncoder) String() string {
-	return "xml"
-}
-
-func NewEncoder() encoder.Encoder {
-	return xmlEncoder{}
-}

config/encoder/yaml/yaml.go

@@ -1,24 +0,0 @@
-package yaml
-
-import (
-	"github.com/ghodss/yaml"
-	"github.com/micro/go-micro/v3/config/encoder"
-)
-
-type yamlEncoder struct{}
-
-func (y yamlEncoder) Encode(v interface{}) ([]byte, error) {
-	return yaml.Marshal(v)
-}
-
-func (y yamlEncoder) Decode(d []byte, v interface{}) error {
-	return yaml.Unmarshal(d, v)
-}
-
-func (y yamlEncoder) String() string {
-	return "yaml"
-}
-
-func NewEncoder() encoder.Encoder {
-	return yamlEncoder{}
-}

config/env/env.go

@@ -0,0 +1,48 @@
+// Package env provides config from environment variables
+package env
+
+import (
+	"encoding/json"
+	"os"
+	"strings"
+
+	"github.com/micro/go-micro/v3/config"
+)
+
+type envConfig struct{}
+
+// NewConfig returns new config
+func NewConfig() (*envConfig, error) {
+	return new(envConfig), nil
+}
+
+func formatKey(v string) string {
+	if len(v) == 0 {
+		return ""
+	}
+
+	v = strings.ToUpper(v)
+	return strings.Replace(v, ".", "_", -1)
+}
+
+func (c *envConfig) Get(path string, options ...config.Option) (config.Value, error) {
+	v := os.Getenv(formatKey(path))
+	if len(v) == 0 {
+		v = "{}"
+	}
+	return config.NewJSONValue([]byte(v)), nil
+}
+
+func (c *envConfig) Set(path string, val interface{}, options ...config.Option) error {
+	key := formatKey(path)
+	v, err := json.Marshal(val)
+	if err != nil {
+		return err
+	}
+	return os.Setenv(key, string(v))
+}
+
+func (c *envConfig) Delete(path string, options ...config.Option) error {
+	v := formatKey(path)
+	return os.Unsetenv(v)
+}

config/loader/loader.go

@@ -1,63 +0,0 @@
-// package loader manages loading from multiple sources
-package loader
-
-import (
-	"context"
-
-	"github.com/micro/go-micro/v3/config/reader"
-	"github.com/micro/go-micro/v3/config/source"
-)
-
-// Loader manages loading sources
-type Loader interface {
-	// Stop the loader
-	Close() error
-	// Load the sources
-	Load(...source.Source) error
-	// A Snapshot of loaded config
-	Snapshot() (*Snapshot, error)
-	// Force sync of sources
-	Sync() error
-	// Watch for changes
-	Watch(...string) (Watcher, error)
-	// Name of loader
-	String() string
-}
-
-// Watcher lets you watch sources and returns a merged ChangeSet
-type Watcher interface {
-	// First call to next may return the current Snapshot
-	// If you are watching a path then only the data from
-	// that path is returned.
-	Next() (*Snapshot, error)
-	// Stop watching for changes
-	Stop() error
-}
-
-// Snapshot is a merged ChangeSet
-type Snapshot struct {
-	// The merged ChangeSet
-	ChangeSet *source.ChangeSet
-	// Deterministic and comparable version of the snapshot
-	Version string
-}
-
-type Options struct {
-	Reader reader.Reader
-	Source []source.Source
-
-	// for alternative data
-	Context context.Context
-}
-
-type Option func(o *Options)
-
-// Copy snapshot
-func Copy(s *Snapshot) *Snapshot {
-	cs := *(s.ChangeSet)
-
-	return &Snapshot{
-		ChangeSet: &cs,
-		Version:   s.Version,
-	}
-}

config/loader/memory/memory.go

@@ -1,459 +0,0 @@
-package memory
-
-import (
-	"bytes"
-	"container/list"
-	"errors"
-	"fmt"
-	"strings"
-	"sync"
-	"time"
-
-	"github.com/micro/go-micro/v3/config/loader"
-	"github.com/micro/go-micro/v3/config/reader"
-	"github.com/micro/go-micro/v3/config/reader/json"
-	"github.com/micro/go-micro/v3/config/source"
-)
-
-type memory struct {
-	exit chan bool
-	opts loader.Options
-
-	sync.RWMutex
-	// the current snapshot
-	snap *loader.Snapshot
-	// the current values
-	vals reader.Values
-	// all the sets
-	sets []*source.ChangeSet
-	// all the sources
-	sources []source.Source
-
-	watchers *list.List
-}
-
-type updateValue struct {
-	version string
-	value   reader.Value
-}
-
-type watcher struct {
-	exit    chan bool
-	path    []string
-	value   reader.Value
-	reader  reader.Reader
-	version string
-	updates chan updateValue
-}
-
-func (m *memory) watch(idx int, s source.Source) {
-	// watches a source for changes
-	watch := func(idx int, s source.Watcher) error {
-		for {
-			// get changeset
-			cs, err := s.Next()
-			if err != nil {
-				return err
-			}
-
-			m.Lock()
-
-			// save
-			m.sets[idx] = cs
-
-			// merge sets
-			set, err := m.opts.Reader.Merge(m.sets...)
-			if err != nil {
-				m.Unlock()
-				return err
-			}
-
-			// set values
-			m.vals, _ = m.opts.Reader.Values(set)
-			m.snap = &loader.Snapshot{
-				ChangeSet: set,
-				Version:   genVer(),
-			}
-			m.Unlock()
-
-			// send watch updates
-			m.update()
-		}
-	}
-
-	for {
-		// watch the source
-		w, err := s.Watch()
-		if err != nil {
-			time.Sleep(time.Second)
-			continue
-		}
-
-		done := make(chan bool)
-
-		// the stop watch func
-		go func() {
-			select {
-			case <-done:
-			case <-m.exit:
-			}
-			w.Stop()
-		}()
-
-		// block watch
-		if err := watch(idx, w); err != nil {
-			// do something better
-			time.Sleep(time.Second)
-		}
-
-		// close done chan
-		close(done)
-
-		// if the config is closed exit
-		select {
-		case <-m.exit:
-			return
-		default:
-		}
-	}
-}
-
-func (m *memory) loaded() bool {
-	var loaded bool
-	m.RLock()
-	if m.vals != nil {
-		loaded = true
-	}
-	m.RUnlock()
-	return loaded
-}
-
-// reload reads the sets and creates new values
-func (m *memory) reload() error {
-	m.Lock()
-
-	// merge sets
-	set, err := m.opts.Reader.Merge(m.sets...)
-	if err != nil {
-		m.Unlock()
-		return err
-	}
-
-	// set values
-	m.vals, _ = m.opts.Reader.Values(set)
-	m.snap = &loader.Snapshot{
-		ChangeSet: set,
-		Version:   genVer(),
-	}
-
-	m.Unlock()
-
-	// update watchers
-	m.update()
-
-	return nil
-}
-
-func (m *memory) update() {
-	watchers := make([]*watcher, 0, m.watchers.Len())
-
-	m.RLock()
-	for e := m.watchers.Front(); e != nil; e = e.Next() {
-		watchers = append(watchers, e.Value.(*watcher))
-	}
-
-	vals := m.vals
-	snap := m.snap
-	m.RUnlock()
-
-	for _, w := range watchers {
-		if w.version >= snap.Version {
-			continue
-		}
-
-		uv := updateValue{
-			version: m.snap.Version,
-			value:   vals.Get(w.path...),
-		}
-
-		select {
-		case <-w.exit:
-			continue
-		default:
-		}
-		select {
-		case w.updates <- uv:
-		default:
-		}
-	}
-}
-
-// Snapshot returns a snapshot of the current loaded config
-func (m *memory) Snapshot() (*loader.Snapshot, error) {
-	if m.loaded() {
-		m.RLock()
-		snap := loader.Copy(m.snap)
-		m.RUnlock()
-		return snap, nil
-	}
-
-	// not loaded, sync
-	if err := m.Sync(); err != nil {
-		return nil, err
-	}
-
-	// make copy
-	m.RLock()
-	snap := loader.Copy(m.snap)
-	m.RUnlock()
-
-	return snap, nil
-}
-
-// Sync loads all the sources, calls the parser and updates the config
-func (m *memory) Sync() error {
-	//nolint:prealloc
-	var sets []*source.ChangeSet
-
-	m.Lock()
-
-	// read the source
-	var gerr []string
-
-	for _, source := range m.sources {
-		ch, err := source.Read()
-		if err != nil {
-			gerr = append(gerr, err.Error())
-			continue
-		}
-		sets = append(sets, ch)
-	}
-
-	// merge sets
-	set, err := m.opts.Reader.Merge(sets...)
-	if err != nil {
-		m.Unlock()
-		return err
-	}
-
-	// set values
-	vals, err := m.opts.Reader.Values(set)
-	if err != nil {
-		m.Unlock()
-		return err
-	}
-	m.vals = vals
-	m.snap = &loader.Snapshot{
-		ChangeSet: set,
-		Version:   genVer(),
-	}
-
-	m.Unlock()
-
-	// update watchers
-	m.update()
-
-	if len(gerr) > 0 {
-		return fmt.Errorf("source loading errors: %s", strings.Join(gerr, "\n"))
-	}
-
-	return nil
-}
-
-func (m *memory) Close() error {
-	select {
-	case <-m.exit:
-		return nil
-	default:
-		close(m.exit)
-	}
-	return nil
-}
-
-func (m *memory) Get(path ...string) (reader.Value, error) {
-	if !m.loaded() {
-		if err := m.Sync(); err != nil {
-			return nil, err
-		}
-	}
-
-	m.Lock()
-	defer m.Unlock()
-
-	// did sync actually work?
-	if m.vals != nil {
-		return m.vals.Get(path...), nil
-	}
-
-	// assuming vals is nil
-	// create new vals
-
-	ch := m.snap.ChangeSet
-
-	// we are truly screwed, trying to load in a hacked way
-	v, err := m.opts.Reader.Values(ch)
-	if err != nil {
-		return nil, err
-	}
-
-	// lets set it just because
-	m.vals = v
-
-	if m.vals != nil {
-		return m.vals.Get(path...), nil
-	}
-
-	// ok we're going hardcore now
-
-	return nil, errors.New("no values")
-}
-
-func (m *memory) Load(sources ...source.Source) error {
-	var gerrors []string
-
-	for _, source := range sources {
-		set, err := source.Read()
-		if err != nil {
-			gerrors = append(gerrors,
-				fmt.Sprintf("error loading source %s: %v",
-					source,
-					err))
-			// continue processing
-			continue
-		}
-		m.Lock()
-		m.sources = append(m.sources, source)
-		m.sets = append(m.sets, set)
-		idx := len(m.sets) - 1
-		m.Unlock()
-		go m.watch(idx, source)
-	}
-
-	if err := m.reload(); err != nil {
-		gerrors = append(gerrors, err.Error())
-	}
-
-	// Return errors
-	if len(gerrors) != 0 {
-		return errors.New(strings.Join(gerrors, "\n"))
-	}
-	return nil
-}
-
-func (m *memory) Watch(path ...string) (loader.Watcher, error) {
-	value, err := m.Get(path...)
-	if err != nil {
-		return nil, err
-	}
-
-	m.Lock()
-
-	w := &watcher{
-		exit:    make(chan bool),
-		path:    path,
-		value:   value,
-		reader:  m.opts.Reader,
-		updates: make(chan updateValue, 1),
-		version: m.snap.Version,
-	}
-
-	e := m.watchers.PushBack(w)
-
-	m.Unlock()
-
-	go func() {
-		<-w.exit
-		m.Lock()
-		m.watchers.Remove(e)
-		m.Unlock()
-	}()
-
-	return w, nil
-}
-
-func (m *memory) String() string {
-	return "memory"
-}
-
-func (w *watcher) Next() (*loader.Snapshot, error) {
-	update := func(v reader.Value) *loader.Snapshot {
-		w.value = v
-
-		cs := &source.ChangeSet{
-			Data:      v.Bytes(),
-			Format:    w.reader.String(),
-			Source:    "memory",
-			Timestamp: time.Now(),
-		}
-		cs.Checksum = cs.Sum()
-
-		return &loader.Snapshot{
-			ChangeSet: cs,
-			Version:   w.version,
-		}
-
-	}
-
-	for {
-		select {
-		case <-w.exit:
-			return nil, errors.New("watcher stopped")
-
-		case uv := <-w.updates:
-			if uv.version <= w.version {
-				continue
-			}
-
-			v := uv.value
-
-			w.version = uv.version
-
-			if bytes.Equal(w.value.Bytes(), v.Bytes()) {
-				continue
-			}
-
-			return update(v), nil
-		}
-	}
-}
-
-func (w *watcher) Stop() error {
-	select {
-	case <-w.exit:
-	default:
-		close(w.exit)
-		close(w.updates)
-	}
-
-	return nil
-}
-
-func genVer() string {
-	return fmt.Sprintf("%d", time.Now().UnixNano())
-}
-
-func NewLoader(opts ...loader.Option) loader.Loader {
-	options := loader.Options{
-		Reader: json.NewReader(),
-	}
-
-	for _, o := range opts {
-		o(&options)
-	}
-
-	m := &memory{
-		exit:     make(chan bool),
-		opts:     options,
-		watchers: list.New(),
-		sources:  options.Source,
-	}
-
-	m.sets = make([]*source.ChangeSet, len(options.Source))
-
-	for i, s := range options.Source {
-		m.sets[i] = &source.ChangeSet{Source: s.String()}
-		go m.watch(i, s)
-	}
-
-	return m
-}

config/loader/memory/options.go

@@ -1,21 +0,0 @@
-package memory
-
-import (
-	"github.com/micro/go-micro/v3/config/loader"
-	"github.com/micro/go-micro/v3/config/reader"
-	"github.com/micro/go-micro/v3/config/source"
-)
-
-// WithSource appends a source to list of sources
-func WithSource(s source.Source) loader.Option {
-	return func(o *loader.Options) {
-		o.Source = append(o.Source, s)
-	}
-}
-
-// WithReader sets the config reader
-func WithReader(r reader.Reader) loader.Option {
-	return func(o *loader.Options) {
-		o.Reader = r
-	}
-}

config/options.go

@@ -1,28 +0,0 @@
-package config
-
-import (
-	"github.com/micro/go-micro/v3/config/loader"
-	"github.com/micro/go-micro/v3/config/reader"
-	"github.com/micro/go-micro/v3/config/source"
-)
-
-// WithLoader sets the loader for manager config
-func WithLoader(l loader.Loader) Option {
-	return func(o *Options) {
-		o.Loader = l
-	}
-}
-
-// WithSource appends a source to list of sources
-func WithSource(s source.Source) Option {
-	return func(o *Options) {
-		o.Source = append(o.Source, s)
-	}
-}
-
-// WithReader sets the config reader
-func WithReader(r reader.Reader) Option {
-	return func(o *Options) {
-		o.Reader = r
-	}
-}

config/reader/json/json.go

@@ -1,83 +0,0 @@
-package json
-
-import (
-	"errors"
-	"time"
-
-	"github.com/imdario/mergo"
-	"github.com/micro/go-micro/v3/config/encoder"
-	"github.com/micro/go-micro/v3/config/encoder/json"
-	"github.com/micro/go-micro/v3/config/reader"
-	"github.com/micro/go-micro/v3/config/source"
-)
-
-type jsonReader struct {
-	opts reader.Options
-	json encoder.Encoder
-}
-
-func (j *jsonReader) Merge(changes ...*source.ChangeSet) (*source.ChangeSet, error) {
-	var merged map[string]interface{}
-
-	for _, m := range changes {
-		if m == nil {
-			continue
-		}
-
-		if len(m.Data) == 0 {
-			continue
-		}
-
-		codec, ok := j.opts.Encoding[m.Format]
-		if !ok {
-			// fallback
-			codec = j.json
-		}
-
-		var data map[string]interface{}
-		if err := codec.Decode(m.Data, &data); err != nil {
-			return nil, err
-		}
-		if err := mergo.Map(&merged, data, mergo.WithOverride); err != nil {
-			return nil, err
-		}
-	}
-
-	b, err := j.json.Encode(merged)
-	if err != nil {
-		return nil, err
-	}
-
-	cs := &source.ChangeSet{
-		Timestamp: time.Now(),
-		Data:      b,
-		Source:    "json",
-		Format:    j.json.String(),
-	}
-	cs.Checksum = cs.Sum()
-
-	return cs, nil
-}
-
-func (j *jsonReader) Values(ch *source.ChangeSet) (reader.Values, error) {
-	if ch == nil {
-		return nil, errors.New("changeset is nil")
-	}
-	if ch.Format != "json" {
-		return nil, errors.New("unsupported format")
-	}
-	return newValues(ch, j.opts)
-}
-
-func (j *jsonReader) String() string {
-	return "json"
-}
-
-// NewReader creates a json reader
-func NewReader(opts ...reader.Option) reader.Reader {
-	options := reader.NewOptions(opts...)
-	return &jsonReader{
-		json: json.NewEncoder(),
-		opts: options,
-	}
-}

config/reader/json/json_test.go

@@ -1,79 +0,0 @@
-package json
-
-import (
-	"testing"
-
-	"github.com/micro/go-micro/v3/config/reader"
-	"github.com/micro/go-micro/v3/config/source"
-)
-
-func TestReader(t *testing.T) {
-	data := []byte(`{"foo": "bar", "baz": {"bar": "cat"}}`)
-
-	testData := []struct {
-		path  []string
-		value string
-	}{
-		{
-			[]string{"foo"},
-			"bar",
-		},
-		{
-			[]string{"baz", "bar"},
-			"cat",
-		},
-	}
-
-	values := newTestValues(t, data)
-
-	for _, test := range testData {
-		if v := values.Get(test.path...).String(""); v != test.value {
-			t.Fatalf("Expected %s got %s for path %v", test.value, v, test.path)
-		}
-	}
-}
-
-func TestDisableReplaceEnvVars(t *testing.T) {
-	data := []byte(`{"foo": "bar", "baz": {"bar": "test/${test}"}}`)
-
-	tests := []struct {
-		path  []string
-		value string
-		opts  []reader.Option
-	}{
-		{
-			[]string{"baz", "bar"},
-			"test/",
-			nil,
-		},
-		{
-			[]string{"baz", "bar"},
-			"test/${test}",
-			[]reader.Option{reader.WithDisableReplaceEnvVars()},
-		},
-	}
-
-	for _, test := range tests {
-		values := newTestValues(t, data, test.opts...)
-
-		if v := values.Get(test.path...).String(""); v != test.value {
-			t.Fatalf("Expected %s got %s for path %v", test.value, v, test.path)
-		}
-	}
-}
-
-func newTestValues(t *testing.T, data []byte, opts ...reader.Option) reader.Values {
-	r := NewReader(opts...)
-
-	c, err := r.Merge(&source.ChangeSet{Data: data}, &source.ChangeSet{})
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	values, err := r.Values(c)
-	if err != nil {
-		t.Fatal(err)
-	}
-
-	return values
-}

config/reader/json/values.go

@@ -1,206 +0,0 @@
-package json
-
-import (
-	"encoding/json"
-	"fmt"
-	"strconv"
-	"strings"
-	"time"
-
-	simple "github.com/bitly/go-simplejson"
-	"github.com/micro/go-micro/v3/config/reader"
-	"github.com/micro/go-micro/v3/config/source"
-)
-
-type jsonValues struct {
-	ch *source.ChangeSet
-	sj *simple.Json
-}
-
-type jsonValue struct {
-	*simple.Json
-}
-
-func newValues(ch *source.ChangeSet, opts reader.Options) (reader.Values, error) {
-	sj := simple.New()
-	data := ch.Data
-
-	if !opts.DisableReplaceEnvVars {
-		data, _ = reader.ReplaceEnvVars(ch.Data)
-	}
-
-	if err := sj.UnmarshalJSON(data); err != nil {
-		sj.SetPath(nil, string(ch.Data))
-	}
-	return &jsonValues{ch, sj}, nil
-}
-
-func (j *jsonValues) Get(path ...string) reader.Value {
-	return &jsonValue{j.sj.GetPath(path...)}
-}
-
-func (j *jsonValues) Del(path ...string) {
-	// delete the tree?
-	if len(path) == 0 {
-		j.sj = simple.New()
-		return
-	}
-
-	if len(path) == 1 {
-		j.sj.Del(path[0])
-		return
-	}
-
-	vals := j.sj.GetPath(path[:len(path)-1]...)
-	vals.Del(path[len(path)-1])
-	j.sj.SetPath(path[:len(path)-1], vals.Interface())
-	return
-}
-
-func (j *jsonValues) Set(val interface{}, path ...string) {
-	j.sj.SetPath(path, val)
-}
-
-func (j *jsonValues) Bytes() []byte {
-	b, _ := j.sj.MarshalJSON()
-	return b
-}
-
-func (j *jsonValues) Map() map[string]interface{} {
-	m, _ := j.sj.Map()
-	return m
-}
-
-func (j *jsonValues) Scan(v interface{}) error {
-	b, err := j.sj.MarshalJSON()
-	if err != nil {
-		return err
-	}
-	return json.Unmarshal(b, v)
-}
-
-func (j *jsonValues) String() string {
-	return "json"
-}
-
-func (j *jsonValue) Bool(def bool) bool {
-	b, err := j.Json.Bool()
-	if err == nil {
-		return b
-	}
-
-	str, ok := j.Interface().(string)
-	if !ok {
-		return def
-	}
-
-	b, err = strconv.ParseBool(str)
-	if err != nil {
-		return def
-	}
-
-	return b
-}
-
-func (j *jsonValue) Int(def int) int {
-	i, err := j.Json.Int()
-	if err == nil {
-		return i
-	}
-
-	str, ok := j.Interface().(string)
-	if !ok {
-		return def
-	}
-
-	i, err = strconv.Atoi(str)
-	if err != nil {
-		return def
-	}
-
-	return i
-}
-
-func (j *jsonValue) String(def string) string {
-	return j.Json.MustString(def)
-}
-
-func (j *jsonValue) Float64(def float64) float64 {
-	f, err := j.Json.Float64()
-	if err == nil {
-		return f
-	}
-
-	str, ok := j.Interface().(string)
-	if !ok {
-		return def
-	}
-
-	f, err = strconv.ParseFloat(str, 64)
-	if err != nil {
-		return def
-	}
-
-	return f
-}
-
-func (j *jsonValue) Duration(def time.Duration) time.Duration {
-	v, err := j.Json.String()
-	if err != nil {
-		return def
-	}
-
-	value, err := time.ParseDuration(v)
-	if err != nil {
-		return def
-	}
-
-	return value
-}
-
-func (j *jsonValue) StringSlice(def []string) []string {
-	v, err := j.Json.String()
-	if err == nil {
-		sl := strings.Split(v, ",")
-		if len(sl) > 1 {
-			return sl
-		}
-	}
-	return j.Json.MustStringArray(def)
-}
-
-func (j *jsonValue) StringMap(def map[string]string) map[string]string {
-	m, err := j.Json.Map()
-	if err != nil {
-		return def
-	}
-
-	res := map[string]string{}
-
-	for k, v := range m {
-		res[k] = fmt.Sprintf("%v", v)
-	}
-
-	return res
-}
-
-func (j *jsonValue) Scan(v interface{}) error {
-	b, err := j.Json.MarshalJSON()
-	if err != nil {
-		return err
-	}
-	return json.Unmarshal(b, v)
-}
-
-func (j *jsonValue) Bytes() []byte {
-	b, err := j.Json.Bytes()
-	if err != nil {
-		// try return marshalled
-		b, err = j.Json.MarshalJSON()
-		if err != nil {
-			return []byte{}
-		}
-		return b
-	}
-	return b
-}

config/reader/json/values_test.go

@@ -1,86 +0,0 @@
-package json
-
-import (
-	"reflect"
-	"testing"
-
-	"github.com/micro/go-micro/v3/config/reader"
-	"github.com/micro/go-micro/v3/config/source"
-)
-
-func TestValues(t *testing.T) {
-	emptyStr := ""
-	testData := []struct {
-		csdata   []byte
-		path     []string
-		accepter interface{}
-		value    interface{}
-	}{
-		{
-			[]byte(`{"foo": "bar", "baz": {"bar": "cat"}}`),
-			[]string{"foo"},
-			emptyStr,
-			"bar",
-		},
-		{
-			[]byte(`{"foo": "bar", "baz": {"bar": "cat"}}`),
-			[]string{"baz", "bar"},
-			emptyStr,
-			"cat",
-		},
-	}
-
-	for idx, test := range testData {
-		values, err := newValues(&source.ChangeSet{
-			Data: test.csdata,
-		}, reader.Options{})
-		if err != nil {
-			t.Fatal(err)
-		}
-
-		err = values.Get(test.path...).Scan(&test.accepter)
-		if err != nil {
-			t.Fatal(err)
-		}
-		if test.accepter != test.value {
-			t.Fatalf("No.%d Expected %v got %v for path %v", idx, test.value, test.accepter, test.path)
-		}
-	}
-}
-
-func TestStructArray(t *testing.T) {
-	type T struct {
-		Foo string
-	}
-
-	emptyTSlice := []T{}
-
-	testData := []struct {
-		csdata   []byte
-		accepter []T
-		value    []T
-	}{
-		{
-			[]byte(`[{"foo": "bar"}]`),
-			emptyTSlice,
-			[]T{{Foo: "bar"}},
-		},
-	}
-
-	for idx, test := range testData {
-		values, err := newValues(&source.ChangeSet{
-			Data: test.csdata,
-		}, reader.Options{})
-		if err != nil {
-			t.Fatal(err)
-		}
-
-		err = values.Get().Scan(&test.accepter)
-		if err != nil {
-			t.Fatal(err)
-		}
-		if !reflect.DeepEqual(test.accepter, test.value) {
-			t.Fatalf("No.%d Expected %v got %v", idx, test.value, test.accepter)
-		}
-	}
-}

config/reader/options.go

@@ -1,50 +0,0 @@
-package reader
-
-import (
-	"github.com/micro/go-micro/v3/config/encoder"
-	"github.com/micro/go-micro/v3/config/encoder/hcl"
-	"github.com/micro/go-micro/v3/config/encoder/json"
-	"github.com/micro/go-micro/v3/config/encoder/toml"
-	"github.com/micro/go-micro/v3/config/encoder/xml"
-	"github.com/micro/go-micro/v3/config/encoder/yaml"
-)
-
-type Options struct {
-	Encoding              map[string]encoder.Encoder
-	DisableReplaceEnvVars bool
-}
-
-type Option func(o *Options)
-
-func NewOptions(opts ...Option) Options {
-	options := Options{
-		Encoding: map[string]encoder.Encoder{
-			"json": json.NewEncoder(),
-			"yaml": yaml.NewEncoder(),
-			"toml": toml.NewEncoder(),
-			"xml":  xml.NewEncoder(),
-			"hcl":  hcl.NewEncoder(),
-			"yml":  yaml.NewEncoder(),
-		},
-	}
-	for _, o := range opts {
-		o(&options)
-	}
-	return options
-}
-
-func WithEncoder(e encoder.Encoder) Option {
-	return func(o *Options) {
-		if o.Encoding == nil {
-			o.Encoding = make(map[string]encoder.Encoder)
-		}
-		o.Encoding[e.String()] = e
-	}
-}
-
-// WithDisableReplaceEnvVars disables the environment variable interpolation preprocessor
-func WithDisableReplaceEnvVars() Option {
-	return func(o *Options) {
-		o.DisableReplaceEnvVars = true
-	}
-}

config/reader/preprocessor.go

@@ -1,23 +0,0 @@
-package reader
-
-import (
-	"os"
-	"regexp"
-)
-
-func ReplaceEnvVars(raw []byte) ([]byte, error) {
-	re := regexp.MustCompile(`\$\{([A-Za-z0-9_]+)\}`)
-	if re.Match(raw) {
-		dataS := string(raw)
-		res := re.ReplaceAllStringFunc(dataS, replaceEnvVars)
-		return []byte(res), nil
-	} else {
-		return raw, nil
-	}
-}
-
-func replaceEnvVars(element string) string {
-	v := element[2 : len(element)-1]
-	el := os.Getenv(v)
-	return el
-}

config/reader/preprocessor_test.go

@@ -1,73 +0,0 @@
-package reader
-
-import (
-	"os"
-	"strings"
-	"testing"
-)
-
-func TestReplaceEnvVars(t *testing.T) {
-	os.Setenv("myBar", "cat")
-	os.Setenv("MYBAR", "cat")
-	os.Setenv("my_Bar", "cat")
-	os.Setenv("myBar_", "cat")
-
-	testData := []struct {
-		expected string
-		data     []byte
-	}{
-		// Right use cases
-		{
-			`{"foo": "bar", "baz": {"bar": "cat"}}`,
-			[]byte(`{"foo": "bar", "baz": {"bar": "${myBar}"}}`),
-		},
-		{
-			`{"foo": "bar", "baz": {"bar": "cat"}}`,
-			[]byte(`{"foo": "bar", "baz": {"bar": "${MYBAR}"}}`),
-		},
-		{
-			`{"foo": "bar", "baz": {"bar": "cat"}}`,
-			[]byte(`{"foo": "bar", "baz": {"bar": "${my_Bar}"}}`),
-		},
-		{
-			`{"foo": "bar", "baz": {"bar": "cat"}}`,
-			[]byte(`{"foo": "bar", "baz": {"bar": "${myBar_}"}}`),
-		},
-		// Wrong use cases
-		{
-			`{"foo": "bar", "baz": {"bar": "${myBar-}"}}`,
-			[]byte(`{"foo": "bar", "baz": {"bar": "${myBar-}"}}`),
-		},
-		{
-			`{"foo": "bar", "baz": {"bar": "${}"}}`,
-			[]byte(`{"foo": "bar", "baz": {"bar": "${}"}}`),
-		},
-		{
-			`{"foo": "bar", "baz": {"bar": "$sss}"}}`,
-			[]byte(`{"foo": "bar", "baz": {"bar": "$sss}"}}`),
-		},
-		{
-			`{"foo": "bar", "baz": {"bar": "${sss"}}`,
-			[]byte(`{"foo": "bar", "baz": {"bar": "${sss"}}`),
-		},
-		{
-			`{"foo": "bar", "baz": {"bar": "{something}"}}`,
-			[]byte(`{"foo": "bar", "baz": {"bar": "{something}"}}`),
-		},
-		// Use cases without replace env vars
-		{
-			`{"foo": "bar", "baz": {"bar": "cat"}}`,
-			[]byte(`{"foo": "bar", "baz": {"bar": "cat"}}`),
-		},
-	}
-
-	for _, test := range testData {
-		res, err := ReplaceEnvVars(test.data)
-		if err != nil {
-			t.Fatal(err)
-		}
-		if strings.Compare(test.expected, string(res)) != 0 {
-			t.Fatalf("Expected %s got %s", test.expected, res)
-		}
-	}
-}

config/reader/reader.go

@@ -1,38 +0,0 @@
-// Package reader parses change sets and provides config values
-package reader
-
-import (
-	"time"
-
-	"github.com/micro/go-micro/v3/config/source"
-)
-
-// Reader is an interface for merging changesets
-type Reader interface {
-	Merge(...*source.ChangeSet) (*source.ChangeSet, error)
-	Values(*source.ChangeSet) (Values, error)
-	String() string
-}
-
-// Values is returned by the reader
-type Values interface {
-	Bytes() []byte
-	Get(path ...string) Value
-	Set(val interface{}, path ...string)
-	Del(path ...string)
-	Map() map[string]interface{}
-	Scan(v interface{}) error
-}
-
-// Value represents a value of any type
-type Value interface {
-	Bool(def bool) bool
-	Int(def int) int
-	String(def string) string
-	Float64(def float64) float64
-	Duration(def time.Duration) time.Duration
-	StringSlice(def []string) []string
-	StringMap(def map[string]string) map[string]string
-	Scan(val interface{}) error
-	Bytes() []byte
-}

config/secrets/encryption.go

@@ -0,0 +1,70 @@
+package secrets
+
+import (
+	"crypto/aes"
+	"crypto/cipher"
+	"crypto/rand"
+	"encoding/hex"
+	"fmt"
+	"io"
+)
+
+// encrypt/decrypt functions are taken from https://www.melvinvivas.com/how-to-encrypt-and-decrypt-data-using-aes/
+
+func encrypt(stringToEncrypt string, key []byte) (string, error) {
+	plaintext := []byte(stringToEncrypt)
+
+	//Create a new Cipher Block from the key
+	block, err := aes.NewCipher(key)
+	if err != nil {
+		return "", err
+	}
+
+	//Create a new GCM - https://en.wikipedia.org/wiki/Galois/Counter_Mode
+	//https://golang.org/pkg/crypto/cipher/#NewGCM
+	aesGCM, err := cipher.NewGCM(block)
+	if err != nil {
+		return "", err
+	}
+
+	//Create a nonce. Nonce should be from GCM
+	nonce := make([]byte, aesGCM.NonceSize())
+	if _, err = io.ReadFull(rand.Reader, nonce); err != nil {
+		return "", err
+	}
+
+	//Encrypt the data using aesGCM.Seal
+	//Since we don't want to save the nonce somewhere else in this case, we add it as a prefix to the encrypted data. The first nonce argument in Seal is the prefix.
+	ciphertext := aesGCM.Seal(nonce, nonce, plaintext, nil)
+	return fmt.Sprintf("%x", ciphertext), nil
+}
+
+func decrypt(encryptedString string, key []byte) (string, error) {
+	enc, _ := hex.DecodeString(encryptedString)
+
+	//Create a new Cipher Block from the key
+	block, err := aes.NewCipher(key)
+	if err != nil {
+		return "", err
+	}
+
+	//Create a new GCM
+	aesGCM, err := cipher.NewGCM(block)
+	if err != nil {
+		return "", err
+	}
+
+	//Get the nonce size
+	nonceSize := aesGCM.NonceSize()
+
+	//Extract the nonce from the encrypted data
+	nonce, ciphertext := enc[:nonceSize], enc[nonceSize:]
+
+	//Decrypt the data
+	plaintext, err := aesGCM.Open(nil, nonce, ciphertext, nil)
+	if err != nil {
+		return "", err
+	}
+
+	return fmt.Sprintf("%s", plaintext), nil
+}

config/secrets/secrets.go

@@ -0,0 +1,121 @@
+package secrets
+
+import (
+	"encoding/base64"
+	"encoding/json"
+	"fmt"
+
+	"github.com/micro/go-micro/v3/config"
+)
+
+// NewSecrets returns a config that encrypts values at rest
+func NewSecrets(config config.Config, encryptionKey string) (config.Secrets, error) {
+	return newSecrets(config, encryptionKey)
+}
+
+type secretConf struct {
+	config        config.Config
+	encryptionKey string
+}
+
+func newSecrets(config config.Config, encryptionKey string) (*secretConf, error) {
+	return &secretConf{
+		config:        config,
+		encryptionKey: encryptionKey,
+	}, nil
+}
+
+func (c *secretConf) Get(path string, options ...config.Option) (config.Value, error) {
+	val, err := c.config.Get(path, options...)
+	empty := config.NewJSONValue([]byte("null"))
+	if err != nil {
+		return empty, err
+	}
+	var v interface{}
+	err = json.Unmarshal(val.Bytes(), &v)
+	if err != nil {
+		return empty, err
+	}
+	v, err = convertElements(v, c.fromEncrypted)
+	if err != nil {
+		return empty, err
+	}
+	dat, err := json.Marshal(v)
+	if err != nil {
+		return empty, err
+	}
+	return config.NewJSONValue(dat), nil
+}
+
+func (c *secretConf) Set(path string, val interface{}, options ...config.Option) error {
+	// marshal to JSON and back so we can iterate on the
+	// value without reflection
+	JSON, err := json.Marshal(val)
+	if err != nil {
+		return err
+	}
+	var v interface{}
+	err = json.Unmarshal(JSON, &v)
+	if err != nil {
+		return err
+	}
+	v, err = convertElements(v, c.toEncrypted)
+	if err != nil {
+		return err
+	}
+	return c.config.Set(path, v)
+}
+
+func (c *secretConf) Delete(path string, options ...config.Option) error {
+	return c.config.Delete(path, options...)
+}
+
+func convertElements(elem interface{}, conversionFunc func(elem interface{}) (interface{}, error)) (interface{}, error) {
+	switch m := elem.(type) {
+	case map[string]interface{}:
+		for k, v := range m {
+			conv, err := convertElements(v, conversionFunc)
+			if err != nil {
+				return nil, err
+			}
+			m[k] = conv
+
+		}
+		return m, nil
+	}
+
+	return conversionFunc(elem)
+}
+
+func (c *secretConf) toEncrypted(elem interface{}) (interface{}, error) {
+	dat, err := json.Marshal(elem)
+	if err != nil {
+		return nil, err
+	}
+	encrypted, err := encrypt(string(dat), []byte(c.encryptionKey))
+	if err != nil {
+		return nil, fmt.Errorf("Failed to encrypt: %v", err)
+	}
+	return string(base64.StdEncoding.EncodeToString([]byte(encrypted))), nil
+}
+
+func (c *secretConf) fromEncrypted(elem interface{}) (interface{}, error) {
+	s, ok := elem.(string)
+	if !ok {
+		// This bit decides if the Secrets implementation suppports nonencrypted values
+		// ie. we could do:
+		// return nil, fmt.Errorf("Encrypted values should be strings, but got: %v", elem)
+		// but let's go with not making nonencrypted values blow up the whole thing
+		return elem, nil
+	}
+	dec, err := base64.StdEncoding.DecodeString(s)
+	if err != nil {
+		return elem, nil
+	}
+	decrypted, err := decrypt(string(dec), []byte(c.encryptionKey))
+	if err != nil {
+		return elem, nil
+	}
+	var ret interface{}
+	return ret, json.Unmarshal([]byte(decrypted), &ret)
+}

config/source/changeset.go

@@ -1,13 +0,0 @@
-package source
-
-import (
-	"crypto/md5"
-	"fmt"
-)
-
-// Sum returns the md5 checksum of the ChangeSet data
-func (c *ChangeSet) Sum() string {
-	h := md5.New()
-	h.Write(c.Data)
-	return fmt.Sprintf("%x", h.Sum(nil))
-}

config/source/env/README.md

@@ -1,96 +0,0 @@
-# Env Source
-
-The env source reads config from environment variables
-
-## Format
-
-We expect environment variables to be in the standard format of FOO=bar
-
-Keys are converted to lowercase and split on underscore.
-
-
-### Example
-
-```
-DATABASE_ADDRESS=127.0.0.1
-DATABASE_PORT=3306
-```
-
-Becomes
-
-```json
-{
-    "database": {
-        "address": "127.0.0.1",
-        "port": 3306
-    }
-}
-```
-
-## Prefixes
-
-Environment variables can be namespaced so we only have access to a subset. Two options are available:
-
-```
-WithPrefix(p ...string)
-WithStrippedPrefix(p ...string)
-```
-
-The former will preserve the prefix and make it a top level key in the config. The latter eliminates the prefix, reducing the nesting by one. 
-
-#### Example:
-
-Given ENVs of:
-
-```
-APP_DATABASE_ADDRESS=127.0.0.1
-APP_DATABASE_PORT=3306
-VAULT_ADDR=vault:1337
-```
-
-and a source initialized as follows:
-
-```
-src := env.NewSource(
-    env.WithPrefix("VAULT"),
-    env.WithStrippedPrefix("APP"),
-)
-```
-
-The resulting config will be:
-
-```
-{
-    "database": {
-        "address": "127.0.0.1",
-        "port": 3306
-    },
-    "vault": {
-        "addr": "vault:1337"
-    }
-}
-```
-
-
-## New Source
-
-Specify source with data
-
-```go
-src := env.NewSource(
-	// optionally specify prefix
-	env.WithPrefix("MICRO"),
-)
-```
-
-## Load Source
-
-Load the source into config
-
-```go
-// Create new config
-conf := config.NewConfig()
-
-// Load env source
-conf.Load(src)
-```

config/source/env/env.go

@@ -1,146 +0,0 @@
-package env
-
-import (
-	"os"
-	"strconv"
-	"strings"
-	"time"
-
-	"github.com/imdario/mergo"
-	"github.com/micro/go-micro/v3/config/source"
-)
-
-var (
-	DefaultPrefixes = []string{}
-)
-
-type env struct {
-	prefixes         []string
-	strippedPrefixes []string
-	opts             source.Options
-}
-
-func (e *env) Read() (*source.ChangeSet, error) {
-	var changes map[string]interface{}
-
-	for _, env := range os.Environ() {
-
-		if len(e.prefixes) > 0 || len(e.strippedPrefixes) > 0 {
-			notFound := true
-
-			if _, ok := matchPrefix(e.prefixes, env); ok {
-				notFound = false
-			}
-
-			if match, ok := matchPrefix(e.strippedPrefixes, env); ok {
-				env = strings.TrimPrefix(env, match)
-				notFound = false
-			}
-
-			if notFound {
-				continue
-			}
-		}
-
-		pair := strings.SplitN(env, "=", 2)
-		value := pair[1]
-		keys := strings.Split(strings.ToLower(pair[0]), "_")
-		reverse(keys)
-
-		tmp := make(map[string]interface{})
-		for i, k := range keys {
-			if i == 0 {
-				if intValue, err := strconv.Atoi(value); err == nil {
-					tmp[k] = intValue
-				} else if boolValue, err := strconv.ParseBool(value); err == nil {
-					tmp[k] = boolValue
-				} else {
-					tmp[k] = value
-				}
-				continue
-			}
-
-			tmp = map[string]interface{}{k: tmp}
-		}
-
-		if err := mergo.Map(&changes, tmp); err != nil {
-			return nil, err
-		}
-	}
-
-	b, err := e.opts.Encoder.Encode(changes)
-	if err != nil {
-		return nil, err
-	}
-
-	cs := &source.ChangeSet{
-		Format:    e.opts.Encoder.String(),
-		Data:      b,
-		Timestamp: time.Now(),
-		Source:    e.String(),
-	}
-	cs.Checksum = cs.Sum()
-
-	return cs, nil
-}
-
-func matchPrefix(pre []string, s string) (string, bool) {
-	for _, p := range pre {
-		if strings.HasPrefix(s, p) {
-			return p, true
-		}
-	}
-
-	return "", false
-}
-
-func reverse(ss []string) {
-	for i := len(ss)/2 - 1; i >= 0; i-- {
-		opp := len(ss) - 1 - i
-		ss[i], ss[opp] = ss[opp], ss[i]
-	}
-}
-
-func (e *env) Watch() (source.Watcher, error) {
-	return newWatcher()
-}
-
-func (e *env) Write(cs *source.ChangeSet) error {
-	return nil
-}
-
-func (e *env) String() string {
-	return "env"
-}
-
-// NewSource returns a config source for parsing ENV variables.
-// Underscores are delimiters for nesting, and all keys are lowercased.
-//
-// Example:
-//      "DATABASE_SERVER_HOST=localhost" will convert to
-//
-//      {
-//          "database": {
-//              "server": {
-//                  "host": "localhost"
-//              }
-//          }
-//      }
-func NewSource(opts ...source.Option) source.Source {
-	options := source.NewOptions(opts...)
-
-	var sp []string
-	var pre []string
-	if p, ok := options.Context.Value(strippedPrefixKey{}).([]string); ok {
-		sp = p
-	}
-
-	if p, ok := options.Context.Value(prefixKey{}).([]string); ok {
-		pre = p
-	}
-
-	if len(sp) > 0 || len(pre) > 0 {
-		pre = append(pre, DefaultPrefixes...)
-	}
-	return &env{prefixes: pre, strippedPrefixes: sp, opts: options}
-}

config/source/env/env_test.go

@@ -1,112 +0,0 @@
-package env
-
-import (
-	"encoding/json"
-	"os"
-	"testing"
-	"time"
-
-	"github.com/micro/go-micro/v3/config/source"
-)
-
-func TestEnv_Read(t *testing.T) {
-	expected := map[string]map[string]string{
-		"database": {
-			"host":       "localhost",
-			"password":   "password",
-			"datasource": "user:password@tcp(localhost:port)/db?charset=utf8mb4&parseTime=True&loc=Local",
-		},
-	}
-
-	os.Setenv("DATABASE_HOST", "localhost")
-	os.Setenv("DATABASE_PASSWORD", "password")
-	os.Setenv("DATABASE_DATASOURCE", "user:password@tcp(localhost:port)/db?charset=utf8mb4&parseTime=True&loc=Local")
-
-	source := NewSource()
-	c, err := source.Read()
-	if err != nil {
-		t.Error(err)
-	}
-
-	var actual map[string]interface{}
-	if err := json.Unmarshal(c.Data, &actual); err != nil {
-		t.Error(err)
-	}
-
-	actualDB := actual["database"].(map[string]interface{})
-
-	for k, v := range expected["database"] {
-		a := actualDB[k]
-
-		if a != v {
-			t.Errorf("expected %v got %v", v, a)
-		}
-	}
-}
-
-func TestEnvvar_Prefixes(t *testing.T) {
-	os.Setenv("APP_DATABASE_HOST", "localhost")
-	os.Setenv("APP_DATABASE_PASSWORD", "password")
-	os.Setenv("VAULT_ADDR", "vault:1337")
-	os.Setenv("MICRO_REGISTRY", "mdns")
-
-	var prefixtests = []struct {
-		prefixOpts   []source.Option
-		expectedKeys []string
-	}{
-		{[]source.Option{WithPrefix("APP", "MICRO")}, []string{"app", "micro"}},
-		{[]source.Option{WithPrefix("MICRO"), WithStrippedPrefix("APP")}, []string{"database", "micro"}},
-		{[]source.Option{WithPrefix("MICRO"), WithStrippedPrefix("APP")}, []string{"database", "micro"}},
-	}
-
-	for _, pt := range prefixtests {
-		source := NewSource(pt.prefixOpts...)
-
-		c, err := source.Read()
-		if err != nil {
-			t.Error(err)
-		}
-
-		var actual map[string]interface{}
-		if err := json.Unmarshal(c.Data, &actual); err != nil {
-			t.Error(err)
-		}
-
-		// assert other prefixes ignored
-		if l := len(actual); l != len(pt.expectedKeys) {
-			t.Errorf("expected %v top keys, got %v", len(pt.expectedKeys), l)
-		}
-
-		for _, k := range pt.expectedKeys {
-			if !containsKey(actual, k) {
-				t.Errorf("expected key %v, not found", k)
-			}
-		}
-	}
-}
-
-func TestEnvvar_WatchNextNoOpsUntilStop(t *testing.T) {
-	src := NewSource(WithStrippedPrefix("GOMICRO_"))
-	w, err := src.Watch()
-	if err != nil {
-		t.Error(err)
-	}
-
-	go func() {
-		time.Sleep(50 * time.Millisecond)
-		w.Stop()
-	}()
-
-	if _, err := w.Next(); err != source.ErrWatcherStopped {
-		t.Errorf("expected watcher stopped error, got %v", err)
-	}
-}
-
-func containsKey(m map[string]interface{}, s string) bool {
-	for k := range m {
-		if k == s {
-			return true
-		}
-	}
-	return false
-}

config/source/env/options.go

@@ -1,50 +0,0 @@
-package env
-
-import (
-	"context"
-
-	"strings"
-
-	"github.com/micro/go-micro/v3/config/source"
-)
-
-type strippedPrefixKey struct{}
-type prefixKey struct{}
-
-// WithStrippedPrefix sets the environment variable prefixes to scope to.
-// These prefixes will be removed from the actual config entries.
-func WithStrippedPrefix(p ...string) source.Option {
-	return func(o *source.Options) {
-		if o.Context == nil {
-			o.Context = context.Background()
-		}
-
-		o.Context = context.WithValue(o.Context, strippedPrefixKey{}, appendUnderscore(p))
-	}
-}
-
-// WithPrefix sets the environment variable prefixes to scope to.
-// These prefixes will not be removed. Each prefix will be considered a top level config entry.
-func WithPrefix(p ...string) source.Option {
-	return func(o *source.Options) {
-		if o.Context == nil {
-			o.Context = context.Background()
-		}
-		o.Context = context.WithValue(o.Context, prefixKey{}, appendUnderscore(p))
-	}
-}
-
-func appendUnderscore(prefixes []string) []string {
-	//nolint:prealloc
-	var result []string
-	for _, p := range prefixes {
-		if !strings.HasSuffix(p, "_") {
-			result = append(result, p+"_")
-			continue
-		}
-
-		result = append(result, p)
-	}
-
-	return result
-}

config/source/env/watcher.go

@@ -1,24 +0,0 @@
-package env
-
-import (
-	"github.com/micro/go-micro/v3/config/source"
-)
-
-type watcher struct {
-	exit chan struct{}
-}
-
-func (w *watcher) Next() (*source.ChangeSet, error) {
-	<-w.exit
-
-	return nil, source.ErrWatcherStopped
-}
-
-func (w *watcher) Stop() error {
-	close(w.exit)
-	return nil
-}
-
-func newWatcher() (source.Watcher, error) {
-	return &watcher{exit: make(chan struct{})}, nil
-}

config/source/etcd/README.md

@@ -1,51 +0,0 @@
-# Etcd Source
-
-The etcd source reads config from etcd key/values
-
-This source supports etcd version 3 and beyond.
-
-## Etcd Format
-
-The etcd source expects keys under the default prefix `/micro/config` (prefix can be changed)
-
-Values are expected to be JSON
-
-```
-// set database
-etcdctl put /micro/config/database '{"address": "10.0.0.1", "port": 3306}'
-// set cache
-etcdctl put /micro/config/cache '{"address": "10.0.0.2", "port": 6379}'
-```
-
-Keys are split on `/` so access becomes
-
-```
-conf.Get("micro", "config", "database")
-```
-
-## New Source
-
-Specify source with data
-
-```go
-etcdSource := etcd.NewSource(
-	// optionally specify etcd address; default to localhost:8500
-	etcd.WithAddress("10.0.0.10:8500"),
-	// optionally specify prefix; defaults to /micro/config
-	etcd.WithPrefix("/my/prefix"),
-	// optionally strip the provided prefix from the keys, defaults to false
-	etcd.StripPrefix(true),
-)
-```
-
-## Load Source
-
-Load the source into config
-
-```go
-// Create new config
-conf := config.NewConfig()
-
-// Load file source
-conf.Load(etcdSource)
-```

config/source/etcd/etcd.go

@@ -1,145 +0,0 @@
-package etcd
-
-import (
-	"context"
-	"fmt"
-	"net"
-	"time"
-
-	cetcd "github.com/coreos/etcd/clientv3"
-	"github.com/coreos/etcd/mvcc/mvccpb"
-	"github.com/micro/go-micro/v3/config/source"
-)
-
-// Currently a single etcd reader
-type etcd struct {
-	prefix      string
-	stripPrefix string
-	opts        source.Options
-	client      *cetcd.Client
-	cerr        error
-}
-
-var (
-	DefaultPrefix = "/micro/config/"
-)
-
-func (c *etcd) Read() (*source.ChangeSet, error) {
-	if c.cerr != nil {
-		return nil, c.cerr
-	}
-
-	rsp, err := c.client.Get(context.Background(), c.prefix, cetcd.WithPrefix())
-	if err != nil {
-		return nil, err
-	}
-
-	if rsp == nil || len(rsp.Kvs) == 0 {
-		return nil, fmt.Errorf("source not found: %s", c.prefix)
-	}
-
-	kvs := make([]*mvccpb.KeyValue, 0, len(rsp.Kvs))
-	for _, v := range rsp.Kvs {
-		kvs = append(kvs, (*mvccpb.KeyValue)(v))
-	}
-
-	data := makeMap(c.opts.Encoder, kvs, c.stripPrefix)
-
-	b, err := c.opts.Encoder.Encode(data)
-	if err != nil {
-		return nil, fmt.Errorf("error reading source: %v", err)
-	}
-
-	cs := &source.ChangeSet{
-		Timestamp: time.Now(),
-		Source:    c.String(),
-		Data:      b,
-		Format:    c.opts.Encoder.String(),
-	}
-	cs.Checksum = cs.Sum()
-
-	return cs, nil
-}
-
-func (c *etcd) String() string {
-	return "etcd"
-}
-
-func (c *etcd) Watch() (source.Watcher, error) {
-	if c.cerr != nil {
-		return nil, c.cerr
-	}
-	cs, err := c.Read()
-	if err != nil {
-		return nil, err
-	}
-	return newWatcher(c.prefix, c.stripPrefix, c.client.Watcher, cs, c.opts)
-}
-
-func (c *etcd) Write(cs *source.ChangeSet) error {
-	return nil
-}
-
-func NewSource(opts ...source.Option) source.Source {
-	options := source.NewOptions(opts...)
-
-	var endpoints []string
-
-	// check if there are any addrs
-	addrs, ok := options.Context.Value(addressKey{}).([]string)
-	if ok {
-		for _, a := range addrs {
-			addr, port, err := net.SplitHostPort(a)
-			if ae, ok := err.(*net.AddrError); ok && ae.Err == "missing port in address" {
-				port = "2379"
-				addr = a
-				endpoints = append(endpoints, fmt.Sprintf("%s:%s", addr, port))
-			} else if err == nil {
-				endpoints = append(endpoints, fmt.Sprintf("%s:%s", addr, port))
-			}
-		}
-	}
-
-	if len(endpoints) == 0 {
-		endpoints = []string{"localhost:2379"}
-	}
-
-	// check dial timeout option
-	dialTimeout, ok := options.Context.Value(dialTimeoutKey{}).(time.Duration)
-	if !ok {
-		dialTimeout = 3 * time.Second // default dial timeout
-	}
-
-	config := cetcd.Config{
-		Endpoints:   endpoints,
-		DialTimeout: dialTimeout,
-	}
-
-	u, ok := options.Context.Value(authKey{}).(*authCreds)
-	if ok {
-		config.Username = u.Username
-		config.Password = u.Password
-	}
-
-	// use default config
-	client, err := cetcd.New(config)
-
-	prefix := DefaultPrefix
-	sp := ""
-	f, ok := options.Context.Value(prefixKey{}).(string)
-	if ok {
-		prefix = f
-	}
-
-	if b, ok := options.Context.Value(stripPrefixKey{}).(bool); ok && b {
-		sp = prefix
-	}
-
-	return &etcd{
-		prefix:      prefix,
-		stripPrefix: sp,
-		opts:        options,
-		client:      client,
-		cerr:        err,
-	}
-}

config/source/etcd/options.go

@@ -1,70 +0,0 @@
-package etcd
-
-import (
-	"context"
-	"time"
-
-	"github.com/micro/go-micro/v3/config/source"
-)
-
-type addressKey struct{}
-type prefixKey struct{}
-type stripPrefixKey struct{}
-type authKey struct{}
-type dialTimeoutKey struct{}
-
-type authCreds struct {
-	Username string
-	Password string
-}
-
-// WithAddress sets the etcd address
-func WithAddress(a ...string) source.Option {
-	return func(o *source.Options) {
-		if o.Context == nil {
-			o.Context = context.Background()
-		}
-		o.Context = context.WithValue(o.Context, addressKey{}, a)
-	}
-}
-
-// WithPrefix sets the key prefix to use
-func WithPrefix(p string) source.Option {
-	return func(o *source.Options) {
-		if o.Context == nil {
-			o.Context = context.Background()
-		}
-		o.Context = context.WithValue(o.Context, prefixKey{}, p)
-	}
-}
-
-// StripPrefix indicates whether to remove the prefix from config entries, or leave it in place.
-func StripPrefix(strip bool) source.Option {
-	return func(o *source.Options) {
-		if o.Context == nil {
-			o.Context = context.Background()
-		}
-
-		o.Context = context.WithValue(o.Context, stripPrefixKey{}, strip)
-	}
-}
-
-// Auth allows you to specify username/password
-func Auth(username, password string) source.Option {
-	return func(o *source.Options) {
-		if o.Context == nil {
-			o.Context = context.Background()
-		}
-		o.Context = context.WithValue(o.Context, authKey{}, &authCreds{Username: username, Password: password})
-	}
-}
-
-// WithDialTimeout set the time out for dialing to etcd
-func WithDialTimeout(timeout time.Duration) source.Option {
-	return func(o *source.Options) {
-		if o.Context == nil {
-			o.Context = context.Background()
-		}
-		o.Context = context.WithValue(o.Context, dialTimeoutKey{}, timeout)
-	}
-}

config/source/etcd/util.go

@@ -1,89 +0,0 @@
-package etcd
-
-import (
-	"strings"
-
-	"github.com/coreos/etcd/clientv3"
-	"github.com/coreos/etcd/mvcc/mvccpb"
-	"github.com/micro/go-micro/v3/config/encoder"
-)
-
-func makeEvMap(e encoder.Encoder, data map[string]interface{}, kv []*clientv3.Event, stripPrefix string) map[string]interface{} {
-	if data == nil {
-		data = make(map[string]interface{})
-	}
-
-	for _, v := range kv {
-		switch mvccpb.Event_EventType(v.Type) {
-		case mvccpb.DELETE:
-			data = update(e, data, (*mvccpb.KeyValue)(v.Kv), "delete", stripPrefix)
-		default:
-			data = update(e, data, (*mvccpb.KeyValue)(v.Kv), "insert", stripPrefix)
-		}
-	}
-
-	return data
-}
-
-func makeMap(e encoder.Encoder, kv []*mvccpb.KeyValue, stripPrefix string) map[string]interface{} {
-	data := make(map[string]interface{})
-
-	for _, v := range kv {
-		data = update(e, data, v, "put", stripPrefix)
-	}
-
-	return data
-}
-
-func update(e encoder.Encoder, data map[string]interface{}, v *mvccpb.KeyValue, action, stripPrefix string) map[string]interface{} {
-	// remove prefix if non empty, and ensure leading / is removed as well
-	vkey := strings.TrimPrefix(strings.TrimPrefix(string(v.Key), stripPrefix), "/")
-	// split on prefix
-	haveSplit := strings.Contains(vkey, "/")
-	keys := strings.Split(vkey, "/")
-
-	var vals interface{}
-	e.Decode(v.Value, &vals)
-
-	if !haveSplit && len(keys) == 1 {
-		switch action {
-		case "delete":
-			data = make(map[string]interface{})
-		default:
-			v, ok := vals.(map[string]interface{})
-			if ok {
-				data = v
-			}
-		}
-		return data
-	}
-
-	// set data for first iteration
-	kvals := data
-	// iterate the keys and make maps
-	for i, k := range keys {
-		kval, ok := kvals[k].(map[string]interface{})
-		if !ok {
-			// create next map
-			kval = make(map[string]interface{})
-			// set it
-			kvals[k] = kval
-		}
-
-		// last key: write vals
-		if l := len(keys) - 1; i == l {
-			switch action {
-			case "delete":
-				delete(kvals, k)
-			default:
-				kvals[k] = vals
-			}
-			break
-		}
-
-		// set kvals for next iterator
-		kvals = kval
-	}
-
-	return data
-}

config/source/etcd/watcher.go

@@ -1,113 +0,0 @@
-package etcd
-
-import (
-	"context"
-	"errors"
-	"sync"
-	"time"
-
-	cetcd "github.com/coreos/etcd/clientv3"
-	"github.com/micro/go-micro/v3/config/source"
-)
-
-type watcher struct {
-	opts        source.Options
-	name        string
-	stripPrefix string
-
-	sync.RWMutex
-	cs *source.ChangeSet
-
-	ch   chan *source.ChangeSet
-	exit chan bool
-}
-
-func newWatcher(key, strip string, wc cetcd.Watcher, cs *source.ChangeSet, opts source.Options) (source.Watcher, error) {
-	w := &watcher{
-		opts:        opts,
-		name:        "etcd",
-		stripPrefix: strip,
-		cs:          cs,
-		ch:          make(chan *source.ChangeSet),
-		exit:        make(chan bool),
-	}
-
-	ch := wc.Watch(context.Background(), key, cetcd.WithPrefix())
-
-	go w.run(wc, ch)
-
-	return w, nil
-}
-
-func (w *watcher) handle(evs []*cetcd.Event) {
-	w.RLock()
-	data := w.cs.Data
-	w.RUnlock()
-
-	var vals map[string]interface{}
-
-	// unpackage existing changeset
-	if err := w.opts.Encoder.Decode(data, &vals); err != nil {
-		return
-	}
-
-	// update base changeset
-	d := makeEvMap(w.opts.Encoder, vals, evs, w.stripPrefix)
-
-	// pack the changeset
-	b, err := w.opts.Encoder.Encode(d)
-	if err != nil {
-		return
-	}
-
-	// create new changeset
-	cs := &source.ChangeSet{
-		Timestamp: time.Now(),
-		Source:    w.name,
-		Data:      b,
-		Format:    w.opts.Encoder.String(),
-	}
-	cs.Checksum = cs.Sum()
-
-	// set base change set
-	w.Lock()
-	w.cs = cs
-	w.Unlock()
-
-	// send update
-	w.ch <- cs
-}
-
-func (w *watcher) run(wc cetcd.Watcher, ch cetcd.WatchChan) {
-	for {
-		select {
-		case rsp, ok := <-ch:
-			if !ok {
-				return
-			}
-			w.handle(rsp.Events)
-		case <-w.exit:
-			wc.Close()
-			return
-		}
-	}
-}
-
-func (w *watcher) Next() (*source.ChangeSet, error) {
-	select {
-	case cs := <-w.ch:
-		return cs, nil
-	case <-w.exit:
-		return nil, errors.New("watcher stopped")
-	}
-}
-
-func (w *watcher) Stop() error {
-	select {
-	case <-w.exit:
-		return nil
-	default:
-		close(w.exit)
-	}
-	return nil
-}

config/source/file/README.md

@@ -1,70 +0,0 @@
-# File Source
-
-The file source reads config from a file. 
-
-It uses the File extension to determine the Format e.g `config.yaml` has the yaml format. 
-It does not make use of encoders or interpet the file data. If a file extension is not present 
-the source Format will default to the Encoder in options.
-
-## Example
-
-A config file format in json
-
-```json
-{
-    "hosts": {
-        "database": {
-            "address": "10.0.0.1",
-            "port": 3306
-        },
-        "cache": {
-            "address": "10.0.0.2",
-            "port": 6379
-        }
-    }
-}
-```
-
-## New Source
-
-Specify file source with path to file. Path is optional and will default to `config.json`
-
-```go
-fileSource := file.NewSource(
-	file.WithPath("/tmp/config.json"),
-)
-```
-
-## File Format
-
-To load different file formats e.g yaml, toml, xml simply specify them with their extension
-
-```
-fileSource := file.NewSource(
-        file.WithPath("/tmp/config.yaml"),
-)
-```
-
-If you want to specify a file without extension, ensure you set the encoder to the same format
-
-```
-e := toml.NewEncoder()
-
-fileSource := file.NewSource(
-        file.WithPath("/tmp/config"),
-	source.WithEncoder(e),
-)
-```
-
-## Load Source
-
-Load the source into config
-
-```go
-// Create new config
-conf := config.NewConfig()
-
-// Load file source
-conf.Load(fileSource)
-```
-

config/source/file/file.go

@@ -1,70 +0,0 @@
-// Package file is a file source. Expected format is json
-package file
-
-import (
-	"io/ioutil"
-	"os"
-
-	"github.com/micro/go-micro/v3/config/source"
-)
-
-type file struct {
-	path string
-	data []byte
-	opts source.Options
-}
-
-var (
-	DefaultPath = "config.json"
-)
-
-func (f *file) Read() (*source.ChangeSet, error) {
-	fh, err := os.Open(f.path)
-	if err != nil {
-		return nil, err
-	}
-	defer fh.Close()
-	b, err := ioutil.ReadAll(fh)
-	if err != nil {
-		return nil, err
-	}
-	info, err := fh.Stat()
-	if err != nil {
-		return nil, err
-	}
-
-	cs := &source.ChangeSet{
-		Format:    format(f.path, f.opts.Encoder),
-		Source:    f.String(),
-		Timestamp: info.ModTime(),
-		Data:      b,
-	}
-	cs.Checksum = cs.Sum()
-
-	return cs, nil
-}
-
-func (f *file) String() string {
-	return "file"
-}
-
-func (f *file) Watch() (source.Watcher, error) {
-	if _, err := os.Stat(f.path); err != nil {
-		return nil, err
-	}
-	return newWatcher(f)
-}
-
-func (f *file) Write(cs *source.ChangeSet) error {
-	return nil
-}
-
-func NewSource(opts ...source.Option) source.Source {
-	options := source.NewOptions(opts...)
-	path := DefaultPath
-	f, ok := options.Context.Value(filePathKey{}).(string)
-	if ok {
-		path = f
-	}
-	return &file{opts: options, path: path}
-}

config/source/file/file_test.go

@@ -1,66 +0,0 @@
-package file_test
-
-import (
-	"fmt"
-	"os"
-	"path/filepath"
-	"testing"
-	"time"
-
-	"github.com/micro/go-micro/v3/config"
-	"github.com/micro/go-micro/v3/config/source/file"
-)
-
-func TestConfig(t *testing.T) {
-	data := []byte(`{"foo": "bar"}`)
-	path := filepath.Join(os.TempDir(), fmt.Sprintf("file.%d", time.Now().UnixNano()))
-	fh, err := os.Create(path)
-	if err != nil {
-		t.Error(err)
-	}
-	defer func() {
-		fh.Close()
-		os.Remove(path)
-	}()
-	_, err = fh.Write(data)
-	if err != nil {
-		t.Error(err)
-	}
-
-	conf, err := config.NewConfig()
-	if err != nil {
-		t.Fatal(err)
-	}
-	conf.Load(file.NewSource(file.WithPath(path)))
-	// simulate multiple close
-	go conf.Close()
-	go conf.Close()
-}
-
-func TestFile(t *testing.T) {
-	data := []byte(`{"foo": "bar"}`)
-	path := filepath.Join(os.TempDir(), fmt.Sprintf("file.%d", time.Now().UnixNano()))
-	fh, err := os.Create(path)
-	if err != nil {
-		t.Error(err)
-	}
-	defer func() {
-		fh.Close()
-		os.Remove(path)
-	}()
-
-	_, err = fh.Write(data)
-	if err != nil {
-		t.Error(err)
-	}
-
-	f := file.NewSource(file.WithPath(path))
-	c, err := f.Read()
-	if err != nil {
-		t.Error(err)
-	}
-	if string(c.Data) != string(data) {
-		t.Logf("%+v", c)
-		t.Error("data from file does not match")
-	}
-}

config/source/file/format.go

@@ -1,15 +0,0 @@
-package file
-
-import (
-	"strings"
-
-	"github.com/micro/go-micro/v3/config/encoder"
-)
-
-func format(p string, e encoder.Encoder) string {
-	parts := strings.Split(p, ".")
-	if len(parts) > 1 {
-		return parts[len(parts)-1]
-	}
-	return e.String()
-}

config/source/file/format_test.go

@@ -1,31 +0,0 @@
-package file
-
-import (
-	"testing"
-
-	"github.com/micro/go-micro/v3/config/source"
-)
-
-func TestFormat(t *testing.T) {
-	opts := source.NewOptions()
-	e := opts.Encoder
-
-	testCases := []struct {
-		p string
-		f string
-	}{
-		{"/foo/bar.json", "json"},
-		{"/foo/bar.yaml", "yaml"},
-		{"/foo/bar.xml", "xml"},
-		{"/foo/bar.conf.ini", "ini"},
-		{"conf", e.String()},
-	}
-
-	for _, d := range testCases {
-		f := format(d.p, e)
-		if f != d.f {
-			t.Fatalf("%s: expected %s got %s", d.p, d.f, f)
-		}
-	}
-
-}

config/source/file/options.go

@@ -1,19 +0,0 @@
-package file
-
-import (
-	"context"
-
-	"github.com/micro/go-micro/v3/config/source"
-)
-
-type filePathKey struct{}
-
-// WithPath sets the path to file
-func WithPath(p string) source.Option {
-	return func(o *source.Options) {
-		if o.Context == nil {
-			o.Context = context.Background()
-		}
-		o.Context = context.WithValue(o.Context, filePathKey{}, p)
-	}
-}

config/source/file/watcher.go

@@ -1,77 +0,0 @@
-//+build !linux
-
-package file
-
-import (
-	"os"
-
-	"github.com/fsnotify/fsnotify"
-	"github.com/micro/go-micro/v3/config/source"
-)
-
-type watcher struct {
-	f *file
-
-	fw   *fsnotify.Watcher
-	exit chan bool
-}
-
-func newWatcher(f *file) (source.Watcher, error) {
-	fw, err := fsnotify.NewWatcher()
-	if err != nil {
-		return nil, err
-	}
-
-	fw.Add(f.path)
-
-	return &watcher{
-		f:    f,
-		fw:   fw,
-		exit: make(chan bool),
-	}, nil
-}
-
-func (w *watcher) Next() (*source.ChangeSet, error) {
-	// is it closed?
-	select {
-	case <-w.exit:
-		return nil, source.ErrWatcherStopped
-	default:
-	}
-
-	for {
-		// try get the event
-		select {
-		case event, _ := <-w.fw.Events:
-			if event.Op == fsnotify.Rename {
-				// check existence of file, and add watch again
-				_, err := os.Stat(event.Name)
-				if err == nil || os.IsExist(err) {
-					w.fw.Add(event.Name)
-				}
-			}
-
-			// ARCH: Darwin Kernel Version 18.7.0
-			// ioutil.WriteFile truncates it before writing, but the problem is that
-			// you will receive two events(fsnotify.Chmod and fsnotify.Write).
-			// We can solve this problem by ignoring fsnotify.Chmod event.
-			if event.Op&fsnotify.Write != fsnotify.Write {
-				continue
-			}
-
-			c, err := w.f.Read()
-			if err != nil {
-				return nil, err
-			}
-			return c, nil
-		case err := <-w.fw.Errors:
-			return nil, err
-		case <-w.exit:
-			return nil, source.ErrWatcherStopped
-		}
-	}
-}
-
-func (w *watcher) Stop() error {
-	return w.fw.Close()
-}

config/source/file/watcher_linux.go

@@ -1,80 +0,0 @@
-//+build linux
-
-package file
-
-import (
-	"os"
-	"reflect"
-
-	"github.com/fsnotify/fsnotify"
-	"github.com/micro/go-micro/v3/config/source"
-)
-
-type watcher struct {
-	f *file
-
-	fw   *fsnotify.Watcher
-	exit chan bool
-}
-
-func newWatcher(f *file) (source.Watcher, error) {
-	fw, err := fsnotify.NewWatcher()
-	if err != nil {
-		return nil, err
-	}
-
-	fw.Add(f.path)
-
-	return &watcher{
-		f:    f,
-		fw:   fw,
-		exit: make(chan bool),
-	}, nil
-}
-
-func (w *watcher) Next() (*source.ChangeSet, error) {
-	// is it closed?
-	select {
-	case <-w.exit:
-		return nil, source.ErrWatcherStopped
-	default:
-	}
-
-	for {
-		// try get the event
-		select {
-		case event, _ := <-w.fw.Events:
-			if event.Op == fsnotify.Rename {
-				// check existence of file, and add watch again
-				_, err := os.Stat(event.Name)
-				if err == nil || os.IsExist(err) {
-					w.fw.Add(event.Name)
-				}
-			}
-
-			c, err := w.f.Read()
-			if err != nil {
-				return nil, err
-			}
-
-			// ARCH: Linux centos-7.shared 3.10.0-693.5.2.el7.x86_64
-			// Sometimes, ioutil.WriteFile triggers multiple fsnotify.Write events, which may be a bug.
-
-			// Detect if the file has changed
-			if reflect.DeepEqual(c.Data, w.f.data) {
-				continue
-			}
-			w.f.data = c.Data
-
-			return c, nil
-		case err := <-w.fw.Errors:
-			return nil, err
-		case <-w.exit:
-			return nil, source.ErrWatcherStopped
-		}
-	}
-}
-
-func (w *watcher) Stop() error {
-	return w.fw.Close()
-}

config/source/flag/README.md

@@ -1,47 +0,0 @@
-# Flag Source
-
-The flag source reads config from flags
-
-## Format
-
-We expect the use of the `flag` package. Upper case flags will be lower cased. Dashes will be used as delimiters.
-
-### Example
-
-```
-dbAddress := flag.String("database_address", "127.0.0.1", "the db address")
-dbPort := flag.Int("database_port", 3306, "the db port)
-```
-
-Becomes
-
-```json
-{
-    "database": {
-        "address": "127.0.0.1",
-        "port": 3306
-    }
-}
-```
-
-## New Source
-
-```go
-flagSource := flag.NewSource(
-	// optionally enable reading of unset flags and their default
-	// values into config, defaults to false
-	IncludeUnset(true)
-)
-```
-
-## Load Source
-
-Load the source into config
-
-```go
-// Create new config
-conf := config.NewConfig()
-
-// Load flag source
-conf.Load(flagSource)
-```

config/source/flag/flag.go

@@ -1,101 +0,0 @@
-package flag
-
-import (
-	"errors"
-	"flag"
-	"github.com/imdario/mergo"
-	"github.com/micro/go-micro/v3/config/source"
-	"strings"
-	"time"
-)
-
-type flagsrc struct {
-	opts source.Options
-}
-
-func (fs *flagsrc) Read() (*source.ChangeSet, error) {
-	if !flag.Parsed() {
-		return nil, errors.New("flags not parsed")
-	}
-
-	var changes map[string]interface{}
-
-	visitFn := func(f *flag.Flag) {
-		n := strings.ToLower(f.Name)
-		keys := strings.FieldsFunc(n, split)
-		reverse(keys)
-
-		tmp := make(map[string]interface{})
-		for i, k := range keys {
-			if i == 0 {
-				tmp[k] = f.Value
-				continue
-			}
-
-			tmp = map[string]interface{}{k: tmp}
-		}
-
-		mergo.Map(&changes, tmp) // need to sort error handling
-		return
-	}
-
-	unset, ok := fs.opts.Context.Value(includeUnsetKey{}).(bool)
-	if ok && unset {
-		flag.VisitAll(visitFn)
-	} else {
-		flag.Visit(visitFn)
-	}
-
-	b, err := fs.opts.Encoder.Encode(changes)
-	if err != nil {
-		return nil, err
-	}
-
-	cs := &source.ChangeSet{
-		Format:    fs.opts.Encoder.String(),
-		Data:      b,
-		Timestamp: time.Now(),
-		Source:    fs.String(),
-	}
-	cs.Checksum = cs.Sum()
-
-	return cs, nil
-}
-
-func split(r rune) bool {
-	return r == '-' || r == '_'
-}
-
-func reverse(ss []string) {
-	for i := len(ss)/2 - 1; i >= 0; i-- {
-		opp := len(ss) - 1 - i
-		ss[i], ss[opp] = ss[opp], ss[i]
-	}
-}
-
-func (fs *flagsrc) Watch() (source.Watcher, error) {
-	return source.NewNoopWatcher()
-}
-
-func (fs *flagsrc) Write(cs *source.ChangeSet) error {
-	return nil
-}
-
-func (fs *flagsrc) String() string {
-	return "flag"
-}
-
-// NewSource returns a config source for integrating parsed flags.
-// Hyphens are delimiters for nesting, and all keys are lowercased.
-//
-// Example:
-//      dbhost := flag.String("database-host", "localhost", "the db host name")
-//
-//      {
-//          "database": {
-//              "host": "localhost"
-//          }
-//      }
-func NewSource(opts ...source.Option) source.Source {
-	return &flagsrc{opts: source.NewOptions(opts...)}
-}

config/source/flag/flag_test.go

@@ -1,68 +0,0 @@
-package flag
-
-import (
-	"encoding/json"
-	"flag"
-	"testing"
-)
-
-var (
-	dbuser = flag.String("database-user", "default", "db user")
-	dbhost = flag.String("database-host", "", "db host")
-	dbpw   = flag.String("database-password", "", "db pw")
-)
-
-func initTestFlags() {
-	flag.Set("database-host", "localhost")
-	flag.Set("database-password", "some-password")
-	flag.Parse()
-}
-
-func TestFlagsrc_Read(t *testing.T) {
-	initTestFlags()
-	source := NewSource()
-	c, err := source.Read()
-	if err != nil {
-		t.Error(err)
-	}
-
-	var actual map[string]interface{}
-	if err := json.Unmarshal(c.Data, &actual); err != nil {
-		t.Error(err)
-	}
-
-	actualDB := actual["database"].(map[string]interface{})
-	if actualDB["host"] != *dbhost {
-		t.Errorf("expected %v got %v", *dbhost, actualDB["host"])
-	}
-
-	if actualDB["password"] != *dbpw {
-		t.Errorf("expected %v got %v", *dbpw, actualDB["password"])
-	}
-
-	// unset flags should not be loaded
-	if actualDB["user"] != nil {
-		t.Errorf("expected %v got %v", nil, actualDB["user"])
-	}
-}
-
-func TestFlagsrc_ReadAll(t *testing.T) {
-	initTestFlags()
-	source := NewSource(IncludeUnset(true))
-	c, err := source.Read()
-	if err != nil {
-		t.Error(err)
-	}
-
-	var actual map[string]interface{}
-	if err := json.Unmarshal(c.Data, &actual); err != nil {
-		t.Error(err)
-	}
-
-	actualDB := actual["database"].(map[string]interface{})
-
-	// unset flag defaults should be loaded
-	if actualDB["user"] != *dbuser {
-		t.Errorf("expected %v got %v", *dbuser, actualDB["user"])
-	}
-}

config/source/flag/options.go

@@ -1,20 +0,0 @@
-package flag
-
-import (
-	"context"
-
-	"github.com/micro/go-micro/v3/config/source"
-)
-
-type includeUnsetKey struct{}
-
-// IncludeUnset toggles the loading of unset flags and their respective default values.
-// Default behavior is to ignore any unset flags.
-func IncludeUnset(b bool) source.Option {
-	return func(o *source.Options) {
-		if o.Context == nil {
-			o.Context = context.Background()
-		}
-		o.Context = context.WithValue(o.Context, includeUnsetKey{}, true)
-	}
-}

config/source/memory/README.md

@@ -1,44 +0,0 @@
-# Memory Source
-
-The memory source provides in-memory data as a source
-
-## Memory Format
-
-The expected data format is json
-
-```json
-data := []byte(`{
-    "hosts": {
-        "database": {
-            "address": "10.0.0.1",
-            "port": 3306
-        },
-        "cache": {
-            "address": "10.0.0.2",
-            "port": 6379
-        }
-    }
-}`)
-```
-
-## New Source
-
-Specify source with data
-
-```go
-memorySource := memory.NewSource(
-	memory.WithJSON(data),
-)
-```
-
-## Load Source
-
-Load the source into config
-
-```go
-// Create new config
-conf := config.NewConfig()
-
-// Load memory source
-conf.Load(memorySource)
-```

config/source/memory/memory.go

@@ -1,99 +0,0 @@
-// Package memory is a memory source
-package memory
-
-import (
-	"sync"
-	"time"
-
-	"github.com/google/uuid"
-	"github.com/micro/go-micro/v3/config/source"
-)
-
-type memory struct {
-	sync.RWMutex
-	ChangeSet *source.ChangeSet
-	Watchers  map[string]*watcher
-}
-
-func (s *memory) Read() (*source.ChangeSet, error) {
-	s.RLock()
-	cs := &source.ChangeSet{
-		Format:    s.ChangeSet.Format,
-		Timestamp: s.ChangeSet.Timestamp,
-		Data:      s.ChangeSet.Data,
-		Checksum:  s.ChangeSet.Checksum,
-		Source:    s.ChangeSet.Source,
-	}
-	s.RUnlock()
-	return cs, nil
-}
-
-func (s *memory) Watch() (source.Watcher, error) {
-	w := &watcher{
-		Id:      uuid.New().String(),
-		Updates: make(chan *source.ChangeSet, 100),
-		Source:  s,
-	}
-
-	s.Lock()
-	s.Watchers[w.Id] = w
-	s.Unlock()
-	return w, nil
-}
-
-func (m *memory) Write(cs *source.ChangeSet) error {
-	m.Update(cs)
-	return nil
-}
-
-// Update allows manual updates of the config data.
-func (s *memory) Update(c *source.ChangeSet) {
-	// don't process nil
-	if c == nil {
-		return
-	}
-
-	// hash the file
-	s.Lock()
-	// update changeset
-	s.ChangeSet = &source.ChangeSet{
-		Data:      c.Data,
-		Format:    c.Format,
-		Source:    "memory",
-		Timestamp: time.Now(),
-	}
-	s.ChangeSet.Checksum = s.ChangeSet.Sum()
-
-	// update watchers
-	for _, w := range s.Watchers {
-		select {
-		case w.Updates <- s.ChangeSet:
-		default:
-		}
-	}
-	s.Unlock()
-}
-
-func (s *memory) String() string {
-	return "memory"
-}
-
-func NewSource(opts ...source.Option) source.Source {
-	var options source.Options
-	for _, o := range opts {
-		o(&options)
-	}
-
-	s := &memory{
-		Watchers: make(map[string]*watcher),
-	}
-
-	if options.Context != nil {
-		c, ok := options.Context.Value(changeSetKey{}).(*source.ChangeSet)
-		if ok {
-			s.Update(c)
-		}
-	}
-
-	return s
-}

config/source/memory/options.go

@@ -1,41 +0,0 @@
-package memory
-
-import (
-	"context"
-
-	"github.com/micro/go-micro/v3/config/source"
-)
-
-type changeSetKey struct{}
-
-func withData(d []byte, f string) source.Option {
-	return func(o *source.Options) {
-		if o.Context == nil {
-			o.Context = context.Background()
-		}
-		o.Context = context.WithValue(o.Context, changeSetKey{}, &source.ChangeSet{
-			Data:   d,
-			Format: f,
-		})
-	}
-}
-
-// WithChangeSet allows a changeset to be set
-func WithChangeSet(cs *source.ChangeSet) source.Option {
-	return func(o *source.Options) {
-		if o.Context == nil {
-			o.Context = context.Background()
-		}
-		o.Context = context.WithValue(o.Context, changeSetKey{}, cs)
-	}
-}
-
-// WithJSON allows the source data to be set to json
-func WithJSON(d []byte) source.Option {
-	return withData(d, "json")
-}
-
-// WithYAML allows the source data to be set to yaml
-func WithYAML(d []byte) source.Option {
-	return withData(d, "yaml")
-}

config/source/memory/watcher.go

@@ -1,23 +0,0 @@
-package memory
-
-import (
-	"github.com/micro/go-micro/v3/config/source"
-)
-
-type watcher struct {
-	Id      string
-	Updates chan *source.ChangeSet
-	Source  *memory
-}
-
-func (w *watcher) Next() (*source.ChangeSet, error) {
-	cs := <-w.Updates
-	return cs, nil
-}
-
-func (w *watcher) Stop() error {
-	w.Source.Lock()
-	delete(w.Source.Watchers, w.Id)
-	w.Source.Unlock()
-	return nil
-}

config/source/noop.go

@@ -1,25 +0,0 @@
-package source
-
-import (
-	"errors"
-)
-
-type noopWatcher struct {
-	exit chan struct{}
-}
-
-func (w *noopWatcher) Next() (*ChangeSet, error) {
-	<-w.exit
-
-	return nil, errors.New("noopWatcher stopped")
-}
-
-func (w *noopWatcher) Stop() error {
-	close(w.exit)
-	return nil
-}
-
-// NewNoopWatcher returns a watcher that blocks on Next() until Stop() is called.
-func NewNoopWatcher() (Watcher, error) {
-	return &noopWatcher{exit: make(chan struct{})}, nil
-}

config/source/options.go

@@ -1,49 +0,0 @@
-package source
-
-import (
-	"context"
-
-	"github.com/micro/go-micro/v3/client"
-	"github.com/micro/go-micro/v3/config/encoder"
-	"github.com/micro/go-micro/v3/config/encoder/json"
-)
-
-type Options struct {
-	// Encoder
-	Encoder encoder.Encoder
-
-	// for alternative data
-	Context context.Context
-
-	// Client to use for RPC
-	Client client.Client
-}
-
-type Option func(o *Options)
-
-func NewOptions(opts ...Option) Options {
-	options := Options{
-		Encoder: json.NewEncoder(),
-		Context: context.Background(),
-	}
-
-	for _, o := range opts {
-		o(&options)
-	}
-
-	return options
-}
-
-// WithEncoder sets the source encoder
-func WithEncoder(e encoder.Encoder) Option {
-	return func(o *Options) {
-		o.Encoder = e
-	}
-}
-
-// WithClient sets the source client
-func WithClient(c client.Client) Option {
-	return func(o *Options) {
-		o.Client = c
-	}
-}

config/source/source.go

@@ -1,35 +0,0 @@
-// Package source is the interface for sources
-package source
-
-import (
-	"errors"
-	"time"
-)
-
-var (
-	// ErrWatcherStopped is returned when source watcher has been stopped
-	ErrWatcherStopped = errors.New("watcher stopped")
-)
-
-// Source is the source from which config is loaded
-type Source interface {
-	Read() (*ChangeSet, error)
-	Write(*ChangeSet) error
-	Watch() (Watcher, error)
-	String() string
-}
-
-// ChangeSet represents a set of changes from a source
-type ChangeSet struct {
-	Data      []byte
-	Checksum  string
-	Format    string
-	Source    string
-	Timestamp time.Time
-}
-
-// Watcher watches a source for changes
-type Watcher interface {
-	Next() (*ChangeSet, error)
-	Stop() error
-}

config/store/store.go

@@ -0,0 +1,107 @@
+package store
+
+import (
+	"encoding/json"
+	"strings"
+
+	"github.com/micro/go-micro/v3/config"
+	"github.com/micro/go-micro/v3/store"
+)
+
+// NewConfig returns new config
+func NewConfig(store store.Store, key string) (config.Config, error) {
+	return newConfig(store, key)
+}
+
+type conf struct {
+	key   string
+	store store.Store
+}
+
+func newConfig(store store.Store, key string) (*conf, error) {
+	return &conf{
+		store: store,
+		key:   key,
+	}, nil
+}
+
+func (c *conf) Get(path string, options ...config.Option) (config.Value, error) {
+	rec, err := c.store.Read(c.key)
+	dat := []byte("{}")
+	if err == nil && len(rec) > 0 {
+		dat = rec[0].Value
+	}
+	values := config.NewJSONValues(dat)
+	return values.Get(path), nil
+}
+
+func (c *conf) Set(path string, val interface{}, options ...config.Option) error {
+	rec, err := c.store.Read(c.key)
+	dat := []byte("{}")
+	if err == nil && len(rec) > 0 {
+		dat = rec[0].Value
+	}
+	values := config.NewJSONValues(dat)
+
+	// marshal to JSON and back so we can iterate on the
+	// value without reflection
+	// @todo only do this if a struct
+	JSON, err := json.Marshal(val)
+	if err != nil {
+		return err
+	}
+	var v interface{}
+	err = json.Unmarshal(JSON, &v)
+	if err != nil {
+		return err
+	}
+
+	m, ok := v.(map[string]interface{})
+	if ok {
+		err := traverse(m, []string{path}, func(p string, value interface{}) error {
+			values.Set(p, value)
+			return nil
+		})
+		if err != nil {
+			return err
+		}
+	} else {
+		values.Set(path, val)
+	}
+	return c.store.Write(&store.Record{
+		Key:   c.key,
+		Value: values.Bytes(),
+	})
+}
+
+func traverse(m map[string]interface{}, paths []string, callback func(path string, value interface{}) error) error {
+	for k, v := range m {
+		val, ok := v.(map[string]interface{})
+		if !ok {
+			err := callback(strings.Join(append(paths, k), "."), v)
+			if err != nil {
+				return err
+			}
+			continue
+		}
+		err := traverse(val, append(paths, k), callback)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (c *conf) Delete(path string, options ...config.Option) error {
+	rec, err := c.store.Read(c.key)
+	dat := []byte("{}")
+	if err != nil || len(rec) == 0 {
+		return nil
+	}
+	values := config.NewJSONValues(dat)
+	values.Delete(path)
+	return c.store.Write(&store.Record{
+		Key:   c.key,
+		Value: values.Bytes(),
+	})
+}

config/store/store_test.go

@@ -0,0 +1,119 @@
+package store
+
+import (
+	"reflect"
+	"testing"
+
+	"github.com/micro/go-micro/v3/config"
+	"github.com/micro/go-micro/v3/config/secrets"
+	"github.com/micro/go-micro/v3/store/memory"
+)
+
+type conf1 struct {
+	A string  `json:"a"`
+	B int64   `json:"b"`
+	C float64 `json:"c"`
+	D bool    `json:"d"`
+}
+
+func TestBasics(t *testing.T) {
+	conf, err := NewConfig(memory.NewStore(), "micro")
+	if err != nil {
+		t.Fatal(err)
+	}
+	testBasics(conf, t)
+	// We need to get a new config because existing config so
+	conf, err = NewConfig(memory.NewStore(), "micro1")
+	if err != nil {
+		t.Fatal(err)
+	}
+	secrets, err := secrets.NewSecrets(conf, "somethingRandomButLongEnough32by")
+	if err != nil {
+		t.Fatal(err)
+	}
+	testBasics(secrets, t)
+}
+
+func testBasics(c config.Config, t *testing.T) {
+	original := &conf1{
+		"Hi", int64(42), float64(42.2), true,
+	}
+	err := c.Set("key", original)
+	if err != nil {
+		t.Fatal(err)
+	}
+	getted := &conf1{}
+	val, err := c.Get("key")
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = val.Scan(getted)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !reflect.DeepEqual(original, getted) {
+		t.Fatalf("Not equal: %v and %v", original, getted)
+	}
+
+	// Testing merges now
+	err = c.Set("key", map[string]interface{}{
+		"b": 55,
+		"e": map[string]interface{}{
+			"e1": true,
+		},
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	m := map[string]interface{}{}
+	val, err = c.Get("key")
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = val.Scan(&m)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	expected := map[string]interface{}{
+		"a": "Hi",
+		"b": float64(55),
+		"c": float64(42.2),
+		"d": true,
+		"e": map[string]interface{}{
+			"e1": true,
+		},
+	}
+	if !reflect.DeepEqual(m, expected) {
+		t.Fatalf("Not equal: %v and %v", m, expected)
+	}
+
+	// Set just one value
+	expected = map[string]interface{}{
+		"a": "Hi",
+		"b": float64(55),
+		"c": float64(42.2),
+		"d": true,
+		"e": map[string]interface{}{
+			"e1": float64(45),
+		},
+	}
+	err = c.Set("key.e.e1", 45)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	m = map[string]interface{}{}
+	val, err = c.Get("key")
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = val.Scan(&m)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if !reflect.DeepEqual(m, expected) {
+		t.Fatalf("Not equal: %v and %v", m, expected)
+	}
+}

config/value.go

@@ -1,49 +1,215 @@
 package config
 
 import (
+	"encoding/json"
+	"fmt"
+	"strconv"
+	"strings"
 	"time"
 
-	"github.com/micro/go-micro/v3/config/reader"
+	simple "github.com/bitly/go-simplejson"
 )
 
-type value struct{}
+type JSONValues struct {
-
+	values []byte
-func newValue() reader.Value {
+	sj     *simple.Json
-	return new(value)
 }
 
-func (v *value) Bool(def bool) bool {
+type JSONValue struct {
+	*simple.Json
+}
+
+func NewJSONValues(data []byte) *JSONValues {
+	sj := simple.New()
+
+	if err := sj.UnmarshalJSON(data); err != nil {
+		sj.SetPath(nil, string(data))
+	}
+	return &JSONValues{data, sj}
+}
+
+func NewJSONValue(data []byte) *JSONValue {
+	sj := simple.New()
+
+	if err := sj.UnmarshalJSON(data); err != nil {
+		sj.SetPath(nil, string(data))
+	}
+	return &JSONValue{sj}
+}
+
+func (j *JSONValues) Get(path string, options ...Option) Value {
+	paths := strings.Split(path, ".")
+	return &JSONValue{j.sj.GetPath(paths...)}
+}
+
+func (j *JSONValues) Delete(path string, options ...Option) {
+	paths := strings.Split(path, ".")
+	// delete the tree?
+	if len(paths) == 0 {
+		j.sj = simple.New()
+		return
+	}
+
+	if len(paths) == 1 {
+		j.sj.Del(paths[0])
+		return
+	}
+
+	vals := j.sj.GetPath(paths[:len(paths)-1]...)
+	vals.Del(paths[len(paths)-1])
+	j.sj.SetPath(paths[:len(paths)-1], vals.Interface())
+	return
+}
+
+func (j *JSONValues) Set(path string, val interface{}, options ...Option) {
+	paths := strings.Split(path, ".")
+	j.sj.SetPath(paths, val)
+}
+
+func (j *JSONValues) Bytes() []byte {
+	b, _ := j.sj.MarshalJSON()
+	return b
+}
+
+func (j *JSONValues) Map() map[string]interface{} {
+	m, _ := j.sj.Map()
+	return m
+}
+
+func (j *JSONValues) Scan(v interface{}) error {
+	b, err := j.sj.MarshalJSON()
+	if err != nil {
+		return err
+	}
+	return json.Unmarshal(b, v)
+}
+
+func (j *JSONValues) String() string {
+	return "json"
+}
+
+func (j *JSONValue) Bool(def bool) bool {
+	b, err := j.Json.Bool()
+	if err == nil {
+		return b
+	}
+
+	str, ok := j.Interface().(string)
+	if !ok {
+		return def
+	}
+
+	b, err = strconv.ParseBool(str)
+	if err != nil {
+		return def
+	}
+
+	return b
+}
+
+func (j *JSONValue) Int(def int) int {
+	i, err := j.Json.Int()
+	if err == nil {
+		return i
+	}
+
+	str, ok := j.Interface().(string)
+	if !ok {
+		return def
+	}
+
+	i, err = strconv.Atoi(str)
+	if err != nil {
+		return def
+	}
+
+	return i
+}
+
+func (j *JSONValue) String(def string) string {
+	return j.Json.MustString(def)
+}
+
+func (j *JSONValue) Float64(def float64) float64 {
+	f, err := j.Json.Float64()
+	if err == nil {
+		return f
+	}
+
+	str, ok := j.Interface().(string)
+	if !ok {
+		return def
+	}
+
+	f, err = strconv.ParseFloat(str, 64)
+	if err != nil {
+		return def
+	}
+
+	return f
+}
+
+func (j *JSONValue) Duration(def time.Duration) time.Duration {
+	v, err := j.Json.String()
+	if err != nil {
+		return def
+	}
+
+	value, err := time.ParseDuration(v)
+	if err != nil {
+		return def
+	}
+
+	return value
+}
+
+func (j *JSONValue) StringSlice(def []string) []string {
+	v, err := j.Json.String()
+	if err == nil {
+		sl := strings.Split(v, ",")
+		if len(sl) > 1 {
+			return sl
+		}
+	}
+	return j.Json.MustStringArray(def)
+}
+
+func (j *JSONValue) Exists() bool {
 	return false
 }
 
-func (v *value) Int(def int) int {
+func (j *JSONValue) StringMap(def map[string]string) map[string]string {
-	return 0
+	m, err := j.Json.Map()
+	if err != nil {
+		return def
+	}
+
+	res := map[string]string{}
+
+	for k, v := range m {
+		res[k] = fmt.Sprintf("%v", v)
+	}
+
+	return res
 }
 
-func (v *value) String(def string) string {
+func (j *JSONValue) Scan(v interface{}) error {
-	return ""
+	b, err := j.Json.MarshalJSON()
+	if err != nil {
+		return err
+	}
+	return json.Unmarshal(b, v)
 }
 
-func (v *value) Float64(def float64) float64 {
+func (j *JSONValue) Bytes() []byte {
-	return 0.0
+	b, err := j.Json.Bytes()
-}
+	if err != nil {
-
+		// try return marshalled
-func (v *value) Duration(def time.Duration) time.Duration {
+		b, err = j.Json.MarshalJSON()
-	return time.Duration(0)
+		if err != nil {
-}
+			return []byte{}
-
+		}
-func (v *value) StringSlice(def []string) []string {
+		return b
-	return nil
+	}
-}
+	return b
-
-func (v *value) StringMap(def map[string]string) map[string]string {
-	return map[string]string{}
-}
-
-func (v *value) Scan(val interface{}) error {
-	return nil
-}
-
-func (v *value) Bytes() []byte {
-	return nil
 }

debug/debug.go

@@ -1,7 +1,2 @@
-// Package debug provides micro debug packages
+// Package debug provides interfaces for service debugging
 package debug
-
-var (
-	// DefaultName is the name of debug service
-	DefaultName = "go.micro.debug"
-)

events/events.go

@@ -1,4 +1,4 @@
-// Package events contains interfaces for managing events within distributed systems
+// Package events is for event streaming and storage
 package events
 
 import (
@@ -14,18 +14,21 @@ var (
 	ErrEncodingMessage = errors.New("Error encoding message")
 )
 
-// Stream of events
+// Stream is an event streaming interface
 type Stream interface {
-	Publish(topic string, opts ...PublishOption) error
+	Publish(topic string, msg interface{}, opts ...PublishOption) error
-	Subscribe(opts ...SubscribeOption) (<-chan Event, error)
+	Subscribe(topic string, opts ...SubscribeOption) (<-chan Event, error)
 }
 
-// Store of events
+// Store is an event store interface
 type Store interface {
-	Read(opts ...ReadOption) ([]*Event, error)
+	Read(topic string, opts ...ReadOption) ([]*Event, error)
 	Write(event *Event, opts ...WriteOption) error
 }
 
+type AckFunc func() error
+type NackFunc func() error
+
 // Event is the object returned by the broker when you subscribe to a topic
 type Event struct {
 	// ID to uniquely identify the event
@@ -34,13 +37,34 @@ type Event struct {
 	Topic string
 	// Timestamp of the event
 	Timestamp time.Time
-	// Metadata contains the encoded event was indexed by
+	// Metadata contains the values the event was indexed by
 	Metadata map[string]string
 	// Payload contains the encoded message
 	Payload []byte
+
+	ackFunc  AckFunc
+	nackFunc NackFunc
 }
 
 // Unmarshal the events message into an object
 func (e *Event) Unmarshal(v interface{}) error {
 	return json.Unmarshal(e.Payload, v)
 }
+
+// Ack acknowledges successful processing of the event in ManualAck mode
+func (e *Event) Ack() error {
+	return e.ackFunc()
+}
+
+func (e *Event) SetAckFunc(f AckFunc) {
+	e.ackFunc = f
+}
+
+// Nack negatively acknowledges processing of the event (i.e. failure) in ManualAck mode
+func (e *Event) Nack() error {
+	return e.nackFunc()
+}
+
+func (e *Event) SetNackFunc(f NackFunc) {
+	e.nackFunc = f
+}

events/memory/memory_test.go

@@ -1,185 +0,0 @@
-package memory
-
-import (
-	"sync"
-	"testing"
-	"time"
-
-	"github.com/google/uuid"
-	"github.com/micro/go-micro/v3/events"
-	"github.com/stretchr/testify/assert"
-)
-
-type testPayload struct {
-	Message string
-}
-
-func TestStream(t *testing.T) {
-	stream, err := NewStream()
-	assert.Nilf(t, err, "NewStream should not return an error")
-	assert.NotNilf(t, stream, "NewStream should return a stream object")
-
-	// TestMissingTopic will test the topic validation on publish
-	t.Run("TestMissingTopic", func(t *testing.T) {
-		err := stream.Publish("")
-		assert.Equalf(t, err, events.ErrMissingTopic, "Publishing to a blank topic should return an error")
-	})
-
-	// TestFirehose will publish a message to the test topic. The subscriber will subscribe to the
-	// firehose topic (indicated by a lack of the topic option).
-	t.Run("TestFirehose", func(t *testing.T) {
-		payload := &testPayload{Message: "HelloWorld"}
-		metadata := map[string]string{"foo": "bar"}
-
-		// create the subscriber
-		evChan, err := stream.Subscribe()
-		assert.Nilf(t, err, "Subscribe should not return an error")
-
-		// setup the subscriber async
-		var wg sync.WaitGroup
-
-		go func() {
-			timeout := time.NewTimer(time.Millisecond * 250)
-
-			select {
-			case event, _ := <-evChan:
-				assert.NotNilf(t, event, "The message was nil")
-				assert.Equal(t, event.Metadata, metadata, "Metadata didn't match")
-
-				var result testPayload
-				err = event.Unmarshal(&result)
-				assert.Nil(t, err, "Error decoding result")
-				assert.Equal(t, result, *payload, "Payload didn't match")
-
-				wg.Done()
-			case <-timeout.C:
-				t.Fatalf("Event was not recieved")
-			}
-		}()
-
-		err = stream.Publish("test",
-			events.WithPayload(payload),
-			events.WithMetadata(metadata),
-		)
-		assert.Nil(t, err, "Publishing a valid message should not return an error")
-		wg.Add(1)
-
-		// wait for the subscriber to recieve the message or timeout
-		wg.Wait()
-	})
-
-	// TestSubscribeTopic will publish a message to the test topic. The subscriber will subscribe to the
-	// same test topic.
-	t.Run("TestSubscribeTopic", func(t *testing.T) {
-		payload := &testPayload{Message: "HelloWorld"}
-		metadata := map[string]string{"foo": "bar"}
-
-		// create the subscriber
-		evChan, err := stream.Subscribe(events.WithTopic("test"))
-		assert.Nilf(t, err, "Subscribe should not return an error")
-
-		// setup the subscriber async
-		var wg sync.WaitGroup
-
-		go func() {
-			timeout := time.NewTimer(time.Millisecond * 250)
-
-			select {
-			case event, _ := <-evChan:
-				assert.NotNilf(t, event, "The message was nil")
-				assert.Equal(t, event.Metadata, metadata, "Metadata didn't match")
-
-				var result testPayload
-				err = event.Unmarshal(&result)
-				assert.Nil(t, err, "Error decoding result")
-				assert.Equal(t, result, *payload, "Payload didn't match")
-
-				wg.Done()
-			case <-timeout.C:
-				t.Fatalf("Event was not recieved")
-			}
-		}()
-
-		err = stream.Publish("test",
-			events.WithPayload(payload),
-			events.WithMetadata(metadata),
-		)
-		assert.Nil(t, err, "Publishing a valid message should not return an error")
-		wg.Add(1)
-
-		// wait for the subscriber to recieve the message or timeout
-		wg.Wait()
-	})
-
-	// TestSubscribeQueue will publish a message to a random topic. Two subscribers will then consume
-	// the message from the firehose topic with different queues. The second subscriber will be registered
-	// after the message is published to test durability.
-	t.Run("TestSubscribeQueue", func(t *testing.T) {
-		topic := uuid.New().String()
-		payload := &testPayload{Message: "HelloWorld"}
-		metadata := map[string]string{"foo": "bar"}
-
-		// create the first subscriber
-		evChan1, err := stream.Subscribe(events.WithTopic(topic))
-		assert.Nilf(t, err, "Subscribe should not return an error")
-
-		// setup the subscriber async
-		var wg sync.WaitGroup
-
-		go func() {
-			timeout := time.NewTimer(time.Millisecond * 250)
-
-			select {
-			case event, _ := <-evChan1:
-				assert.NotNilf(t, event, "The message was nil")
-				assert.Equal(t, event.Metadata, metadata, "Metadata didn't match")
-
-				var result testPayload
-				err = event.Unmarshal(&result)
-				assert.Nil(t, err, "Error decoding result")
-				assert.Equal(t, result, *payload, "Payload didn't match")
-
-				wg.Done()
-			case <-timeout.C:
-				t.Fatalf("Event was not recieved")
-			}
-		}()
-
-		err = stream.Publish(topic,
-			events.WithPayload(payload),
-			events.WithMetadata(metadata),
-		)
-		assert.Nil(t, err, "Publishing a valid message should not return an error")
-		wg.Add(2)
-
-		// create the second subscriber
-		evChan2, err := stream.Subscribe(
-			events.WithTopic(topic),
-			events.WithQueue("second_queue"),
-			events.WithStartAtTime(time.Now().Add(time.Minute*-1)),
-		)
-		assert.Nilf(t, err, "Subscribe should not return an error")
-
-		go func() {
-			timeout := time.NewTimer(time.Millisecond * 250)
-
-			select {
-			case event, _ := <-evChan2:
-				assert.NotNilf(t, event, "The message was nil")
-				assert.Equal(t, event.Metadata, metadata, "Metadata didn't match")
-
-				var result testPayload
-				err = event.Unmarshal(&result)
-				assert.Nil(t, err, "Error decoding result")
-				assert.Equal(t, result, *payload, "Payload didn't match")
-
-				wg.Done()
-			case <-timeout.C:
-				t.Fatalf("Event was not recieved")
-			}
-		}()
-
-		// wait for the subscriber to recieve the message or timeout
-		wg.Wait()
-	})
-}

events/nats/nats.go

@@ -1,164 +0,0 @@
-package nats
-
-import (
-	"encoding/json"
-	"fmt"
-	"time"
-
-	"github.com/google/uuid"
-	"github.com/nats-io/nats.go"
-	stan "github.com/nats-io/stan.go"
-	"github.com/pkg/errors"
-
-	"github.com/micro/go-micro/v3/events"
-	"github.com/micro/go-micro/v3/logger"
-)
-
-const (
-	defaultClusterID = "micro"
-	eventsTopic      = "events"
-)
-
-// NewStream returns an initialized nats stream or an error if the connection to the nats
-// server could not be established
-func NewStream(opts ...Option) (events.Stream, error) {
-	// parse the options
-	options := Options{
-		ClientID:  uuid.New().String(),
-		ClusterID: defaultClusterID,
-	}
-	for _, o := range opts {
-		o(&options)
-	}
-
-	// connect to nats
-	nopts := nats.GetDefaultOptions()
-	if options.TLSConfig != nil {
-		nopts.Secure = true
-		nopts.TLSConfig = options.TLSConfig
-	}
-	if len(options.Address) > 0 {
-		nopts.Servers = []string{options.Address}
-	}
-	conn, err := nopts.Connect()
-	if err != nil {
-		return nil, fmt.Errorf("Error connecting to nats at %v with tls enabled (%v): %v", options.Address, nopts.TLSConfig != nil, err)
-	}
-
-	// connect to the cluster
-	clusterConn, err := stan.Connect(options.ClusterID, options.ClientID, stan.NatsConn(conn))
-	if err != nil {
-		return nil, fmt.Errorf("Error connecting to nats cluster %v: %v", options.ClusterID, err)
-	}
-
-	return &stream{clusterConn}, nil
-}
-
-type stream struct {
-	conn stan.Conn
-}
-
-// Publish a message to a topic
-func (s *stream) Publish(topic string, opts ...events.PublishOption) error {
-	// validate the topic
-	if len(topic) == 0 {
-		return events.ErrMissingTopic
-	}
-
-	// parse the options
-	options := events.PublishOptions{
-		Timestamp: time.Now(),
-	}
-	for _, o := range opts {
-		o(&options)
-	}
-
-	// encode the message if it's not already encoded
-	var payload []byte
-	if p, ok := options.Payload.([]byte); ok {
-		payload = p
-	} else {
-		p, err := json.Marshal(options.Payload)
-		if err != nil {
-			return events.ErrEncodingMessage
-		}
-		payload = p
-	}
-
-	// construct the event
-	event := &events.Event{
-		ID:        uuid.New().String(),
-		Topic:     topic,
-		Timestamp: options.Timestamp,
-		Metadata:  options.Metadata,
-		Payload:   payload,
-	}
-
-	// serialize the event to bytes
-	bytes, err := json.Marshal(event)
-	if err != nil {
-		return errors.Wrap(err, "Error encoding event")
-	}
-
-	// publish the event to the events channel
-	if _, err := s.conn.PublishAsync(eventsTopic, bytes, nil); err != nil {
-		return errors.Wrap(err, "Error publishing message to events")
-	}
-
-	// publish the event to the topic's channel
-	if _, err := s.conn.PublishAsync(event.Topic, bytes, nil); err != nil {
-		return errors.Wrap(err, "Error publishing message to topic")
-	}
-
-	return nil
-}
-
-// Subscribe to a topic
-func (s *stream) Subscribe(opts ...events.SubscribeOption) (<-chan events.Event, error) {
-	// parse the options
-	options := events.SubscribeOptions{
-		Topic: eventsTopic,
-		Queue: uuid.New().String(),
-	}
-	for _, o := range opts {
-		o(&options)
-	}
-
-	// setup the subscriber
-	c := make(chan events.Event)
-	handleMsg := func(m *stan.Msg) {
-		// decode the message
-		var evt events.Event
-		if err := json.Unmarshal(m.Data, &evt); err != nil {
-			if logger.V(logger.ErrorLevel, logger.DefaultLogger) {
-				logger.Errorf("Error decoding message: %v", err)
-			}
-			// not ackknowledging the message is the way to indicate an error occured
-			return
-		}
-
-		// push onto the channel and wait for the consumer to take the event off before we acknowledge it.
-		c <- evt
-
-		if err := m.Ack(); err != nil && logger.V(logger.ErrorLevel, logger.DefaultLogger) {
-			logger.Errorf("Error acknowledging message: %v", err)
-		}
-	}
-
-	// setup the options
-	subOpts := []stan.SubscriptionOption{
-		stan.DurableName(options.Topic),
-		stan.SetManualAckMode(),
-	}
-	if options.StartAtTime.Unix() > 0 {
-		stan.StartAtTime(options.StartAtTime)
-	}
-
-	// connect the subscriber
-	_, err := s.conn.QueueSubscribe(options.Topic, options.Queue, handleMsg, subOpts...)
-	if err != nil {
-		return nil, errors.Wrap(err, "Error subscribing to topic")
-	}
-
-	return c, nil
-}

events/nats/options.go

@@ -1,42 +0,0 @@
-package nats
-
-import "crypto/tls"
-
-// Options which are used to configure the nats stream
-type Options struct {
-	ClusterID string
-	ClientID  string
-	Address   string
-	TLSConfig *tls.Config
-}
-
-// Option is a function which configures options
-type Option func(o *Options)
-
-// ClusterID sets the cluster id for the nats connection
-func ClusterID(id string) Option {
-	return func(o *Options) {
-		o.ClusterID = id
-	}
-}
-
-// ClientID sets the client id for the nats connection
-func ClientID(id string) Option {
-	return func(o *Options) {
-		o.ClientID = id
-	}
-}
-
-// Address of the nats cluster
-func Address(addr string) Option {
-	return func(o *Options) {
-		o.Address = addr
-	}
-}
-
-// TLSConfig to use when connecting to the cluster
-func TLSConfig(t *tls.Config) Option {
-	return func(o *Options) {
-		o.TLSConfig = t
-	}
-}

events/options.go

@@ -6,9 +6,6 @@ import "time"
 type PublishOptions struct {
 	// Metadata contains any keys which can be used to query the data, for example a customer id
 	Metadata map[string]string
-	// Payload contains any additonal data which is relevent to the event but does not need to be
-	// indexed such as structured data
-	Payload interface{}
 	// Timestamp to set for the event, if the timestamp is a zero value, the current time will be used
 	Timestamp time.Time
 }
@@ -23,13 +20,6 @@ func WithMetadata(md map[string]string) PublishOption {
 	}
 }
 
-// WithPayload sets the payload field on PublishOptions
-func WithPayload(p interface{}) PublishOption {
-	return func(o *PublishOptions) {
-		o.Payload = p
-	}
-}
-
 // WithTimestamp sets the timestamp field on PublishOptions
 func WithTimestamp(t time.Time) PublishOption {
 	return func(o *PublishOptions) {
@@ -42,12 +32,20 @@ type SubscribeOptions struct {
 	// Queue is the name of the subscribers queue, if two subscribers have the same queue the message
 	// should only be published to one of them
 	Queue string
-	// Topic to subscribe to, if left blank the consumer will be subscribed to the firehouse topic which
-	// recieves all events
-	Topic string
 	// StartAtTime is the time from which the messages should be consumed from. If not provided then
 	// the messages will be consumed starting from the moment the Subscription starts.
 	StartAtTime time.Time
+	// AutoAck if true (default true), automatically acknowledges every message so it will not be redelivered.
+	// If false specifies that each message need ts to be manually acknowledged by the subscriber.
+	// If processing is successful the message should be ack'ed to remove the message from the stream.
+	// If processing is unsuccessful the message should be nack'ed (negative acknowledgement) which will mean it will
+	// remain on the stream to be processed again.
+	AutoAck bool
+	AckWait time.Duration
+	// RetryLimit indicates number of times a message is retried
+	RetryLimit int
+	// CustomRetries indicates whether to use RetryLimit
+	CustomRetries bool
 }
 
 // SubscribeOption sets attributes on SubscribeOptions
@@ -60,13 +58,6 @@ func WithQueue(q string) SubscribeOption {
 	}
 }
 
-// WithTopic sets the topic to subscribe to
-func WithTopic(t string) SubscribeOption {
-	return func(o *SubscribeOptions) {
-		o.Topic = t
-	}
-}
-
 // WithStartAtTime sets the StartAtTime field on SubscribeOptions to the value provided
 func WithStartAtTime(t time.Time) SubscribeOption {
 	return func(o *SubscribeOptions) {
@@ -74,6 +65,31 @@ func WithStartAtTime(t time.Time) SubscribeOption {
 	}
 }
 
+// WithAutoAck sets the AutoAck field on SubscribeOptions and an ackWait duration after which if no ack is received
+// the message is requeued in case auto ack is turned off
+func WithAutoAck(ack bool, ackWait time.Duration) SubscribeOption {
+	return func(o *SubscribeOptions) {
+		o.AutoAck = ack
+		o.AckWait = ackWait
+	}
+}
+
+// WithRetryLimit sets the RetryLimit field on SubscribeOptions.
+// Set to -1 for infinite retries (default)
+func WithRetryLimit(retries int) SubscribeOption {
+	return func(o *SubscribeOptions) {
+		o.RetryLimit = retries
+		o.CustomRetries = true
+	}
+}
+
+func (s SubscribeOptions) GetRetryLimit() int {
+	if !s.CustomRetries {
+		return -1
+	}
+	return s.RetryLimit
+}
+
 // WriteOptions contains all the options which can be provided when writing an event to a store
 type WriteOptions struct {
 	// TTL is the duration the event should be recorded for, a zero value TTL indicates the event should
@@ -93,47 +109,24 @@ func WithTTL(d time.Duration) WriteOption {
 
 // ReadOptions contains all the options which can be provided when reading events from a store
 type ReadOptions struct {
-	// Topic to read events from, if no topic is provided events from all topics will be returned
-	Topic string
-	// Query to filter the results using. The store will query the metadata provided when the event
-	// was written to the store
-	Query map[string]string
 	// Limit the number of results to return
-	Limit int
+	Limit uint
 	// Offset the results by this number, useful for paginated queries
-	Offset int
+	Offset uint
 }
 
 // ReadOption sets attributes on ReadOptions
 type ReadOption func(o *ReadOptions)
 
-// ReadTopic sets the topic attribute on ReadOptions
-func ReadTopic(t string) ReadOption {
-	return func(o *ReadOptions) {
-		o.Topic = t
-	}
-}
-
-// ReadFilter sets a key and value in the query
-func ReadFilter(key, value string) ReadOption {
-	return func(o *ReadOptions) {
-		if o.Query == nil {
-			o.Query = map[string]string{key: value}
-		} else {
-			o.Query[key] = value
-		}
-	}
-}
-
 // ReadLimit sets the limit attribute on ReadOptions
-func ReadLimit(l int) ReadOption {
+func ReadLimit(l uint) ReadOption {
 	return func(o *ReadOptions) {
 		o.Limit = 1
 	}
 }
 
 // ReadOffset sets the offset attribute on ReadOptions
-func ReadOffset(l int) ReadOption {
+func ReadOffset(l uint) ReadOption {
 	return func(o *ReadOptions) {
 		o.Offset = 1
 	}

events/store/options.go

@@ -0,0 +1,28 @@
+package store
+
+import (
+	"time"
+
+	"github.com/micro/go-micro/v3/store"
+)
+
+type Options struct {
+	Store store.Store
+	TTL   time.Duration
+}
+
+type Option func(o *Options)
+
+// WithStore sets the underlying store to use
+func WithStore(s store.Store) Option {
+	return func(o *Options) {
+		o.Store = s
+	}
+}
+
+// WithTTL sets the default TTL
+func WithTTL(ttl time.Duration) Option {
+	return func(o *Options) {
+		o.TTL = ttl
+	}
+}

events/store/store.go

@@ -0,0 +1,103 @@
+package store
+
+import (
+	"encoding/json"
+	"time"
+
+	"github.com/micro/go-micro/v3/events"
+	gostore "github.com/micro/go-micro/v3/store"
+	"github.com/micro/go-micro/v3/store/memory"
+	"github.com/pkg/errors"
+)
+
+const joinKey = "/"
+
+// NewStore returns an initialized events store
+func NewStore(opts ...Option) events.Store {
+	// parse the options
+	var options Options
+	for _, o := range opts {
+		o(&options)
+	}
+	if options.TTL.Seconds() == 0 {
+		options.TTL = time.Hour * 24
+	}
+	if options.Store == nil {
+		options.Store = memory.NewStore()
+	}
+
+	// return the store
+	return &evStore{options}
+}
+
+type evStore struct {
+	opts Options
+}
+
+// Read events for a topic
+func (s *evStore) Read(topic string, opts ...events.ReadOption) ([]*events.Event, error) {
+	// validate the topic
+	if len(topic) == 0 {
+		return nil, events.ErrMissingTopic
+	}
+
+	// parse the options
+	options := events.ReadOptions{
+		Offset: 0,
+		Limit:  250,
+	}
+	for _, o := range opts {
+		o(&options)
+	}
+
+	// execute the request
+	recs, err := s.opts.Store.Read(topic+joinKey,
+		gostore.ReadPrefix(),
+		gostore.ReadLimit(options.Limit),
+		gostore.ReadOffset(options.Offset),
+	)
+	if err != nil {
+		return nil, errors.Wrap(err, "Error reading from store")
+	}
+
+	// unmarshal the result
+	result := make([]*events.Event, len(recs))
+	for i, r := range recs {
+		var e events.Event
+		if err := json.Unmarshal(r.Value, &e); err != nil {
+			return nil, errors.Wrap(err, "Invalid event returned from stroe")
+		}
+		result[i] = &e
+	}
+
+	return result, nil
+}
+
+// Write an event to the store
+func (s *evStore) Write(event *events.Event, opts ...events.WriteOption) error {
+	// parse the options
+	options := events.WriteOptions{
+		TTL: s.opts.TTL,
+	}
+	for _, o := range opts {
+		o(&options)
+	}
+
+	// construct the store record
+	bytes, err := json.Marshal(event)
+	if err != nil {
+		return errors.Wrap(err, "Error mashaling event to JSON")
+	}
+	record := &gostore.Record{
+		Key:    event.Topic + joinKey + event.ID,
+		Value:  bytes,
+		Expiry: options.TTL,
+	}
+
+	// write the record to the store
+	if err := s.opts.Store.Write(record); err != nil {
+		return errors.Wrap(err, "Error writing to the store")
+	}
+
+	return nil
+}

events/store/store_test.go

@@ -0,0 +1,48 @@
+package store
+
+import (
+	"testing"
+
+	"github.com/google/uuid"
+	"github.com/micro/go-micro/v3/events"
+	"github.com/stretchr/testify/assert"
+)
+
+func TestStore(t *testing.T) {
+	store := NewStore()
+
+	testData := []events.Event{
+		{ID: uuid.New().String(), Topic: "foo"},
+		{ID: uuid.New().String(), Topic: "foo"},
+		{ID: uuid.New().String(), Topic: "bar"},
+	}
+
+	// write the records to the store
+	t.Run("Write", func(t *testing.T) {
+		for _, event := range testData {
+			err := store.Write(&event)
+			assert.Nilf(t, err, "Writing an event should not return an error")
+		}
+	})
+
+	// should not be able to read events from a blank topic
+	t.Run("ReadMissingTopic", func(t *testing.T) {
+		evs, err := store.Read("")
+		assert.Equal(t, err, events.ErrMissingTopic, "Reading a blank topic should return an error")
+		assert.Nil(t, evs, "No events should be returned")
+	})
+
+	// should only get the events from the topic requested
+	t.Run("ReadTopic", func(t *testing.T) {
+		evs, err := store.Read("foo")
+		assert.Nilf(t, err, "No error should be returned")
+		assert.Len(t, evs, 2, "Only the events for this topic should be returned")
+	})
+
+	// limits should be honoured
+	t.Run("ReadTopicLimit", func(t *testing.T) {
+		evs, err := store.Read("foo", events.ReadLimit(1))
+		assert.Nilf(t, err, "No error should be returned")
+		assert.Len(t, evs, 1, "The result should include no more than the read limit")
+	})
+}

events/stream/memory/memory.go

@@ -32,6 +32,12 @@ type subscriber struct {
 	Queue   string
 	Topic   string
 	Channel chan events.Event
+
+	sync.RWMutex
+	retryMap   map[string]int
+	retryLimit int
+	autoAck    bool
+	ackWait    time.Duration
 }
 
 type mem struct {
@@ -41,7 +47,7 @@ type mem struct {
 	sync.RWMutex
 }
 
-func (m *mem) Publish(topic string, opts ...events.PublishOption) error {
+func (m *mem) Publish(topic string, msg interface{}, opts ...events.PublishOption) error {
 	// validate the topic
 	if len(topic) == 0 {
 		return events.ErrMissingTopic
@@ -57,10 +63,10 @@ func (m *mem) Publish(topic string, opts ...events.PublishOption) error {
 
 	// encode the message if it's not already encoded
 	var payload []byte
-	if p, ok := options.Payload.([]byte); ok {
+	if p, ok := msg.([]byte); ok {
 		payload = p
 	} else {
-		p, err := json.Marshal(options.Payload)
+		p, err := json.Marshal(msg)
 		if err != nil {
 			return events.ErrEncodingMessage
 		}
@@ -94,20 +100,38 @@ func (m *mem) Publish(topic string, opts ...events.PublishOption) error {
 	return nil
 }
 
-func (m *mem) Subscribe(opts ...events.SubscribeOption) (<-chan events.Event, error) {
+func (m *mem) Subscribe(topic string, opts ...events.SubscribeOption) (<-chan events.Event, error) {
+	// validate the topic
+	if len(topic) == 0 {
+		return nil, events.ErrMissingTopic
+	}
+
 	// parse the options
 	options := events.SubscribeOptions{
-		Queue: uuid.New().String(),
+		Queue:   uuid.New().String(),
+		AutoAck: true,
 	}
 	for _, o := range opts {
 		o(&options)
 	}
+	// TODO RetryLimit
 
 	// setup the subscriber
 	sub := &subscriber{
-		Channel: make(chan events.Event),
+		Channel:    make(chan events.Event),
-		Topic:   options.Topic,
+		Topic:      topic,
-		Queue:   options.Queue,
+		Queue:      options.Queue,
+		retryMap:   map[string]int{},
+		autoAck:    true,
+		retryLimit: options.GetRetryLimit(),
+	}
+
+	if !options.AutoAck {
+		if options.AckWait == 0 {
+			return nil, fmt.Errorf("invalid AckWait passed, should be positive integer")
+		}
+		sub.autoAck = options.AutoAck
+		sub.ackWait = options.AckWait
 	}
 
 	// register the subscriber
@@ -124,16 +148,11 @@ func (m *mem) Subscribe(opts ...events.SubscribeOption) (<-chan events.Event, er
 	return sub.Channel, nil
 }
 
-// lookupPreviousEvents finds events for a subscriber which occured before a given time and sends
+// lookupPreviousEvents finds events for a subscriber which occurred before a given time and sends
 // them into the subscribers channel
 func (m *mem) lookupPreviousEvents(sub *subscriber, startTime time.Time) {
-	var prefix string
-	if len(sub.Topic) > 0 {
-		prefix = sub.Topic + "/"
-	}
-
 	// lookup all events which match the topic (a blank topic will return all results)
-	recs, err := m.store.Read(prefix, store.ReadPrefix())
+	recs, err := m.store.Read(sub.Topic+"/", store.ReadPrefix())
 	if err != nil && logger.V(logger.ErrorLevel, logger.DefaultLogger) {
 		logger.Errorf("Error looking up previous events: %v", err)
 		return
@@ -150,8 +169,7 @@ func (m *mem) lookupPreviousEvents(sub *subscriber, startTime time.Time) {
 		if ev.Timestamp.Unix() < startTime.Unix() {
 			continue
 		}
-
+		sendEvent(&ev, sub)
-		sub.Channel <- ev
 	}
 }
 
@@ -173,9 +191,60 @@ func (m *mem) handleEvent(ev *events.Event) {
 	}
 
 	// send the message to each channel async (since one channel might be blocked)
-	for _, sub := range subs {
+	for _, sub := range filteredSubs {
-		go func(s *subscriber) {
+		sendEvent(ev, sub)
-			s.Channel <- *ev
+	}
-		}(sub)
+}
+
+func sendEvent(ev *events.Event, sub *subscriber) {
+	go func(s *subscriber) {
+		evCopy := *ev
+		if s.autoAck {
+			s.Channel <- evCopy
+			return
+		}
+		evCopy.SetAckFunc(ackFunc(s, evCopy))
+		evCopy.SetNackFunc(nackFunc(s, evCopy))
+		s.retryMap[evCopy.ID] = 0
+		tick := time.NewTicker(s.ackWait)
+		defer tick.Stop()
+		for range tick.C {
+			s.Lock()
+			count, ok := s.retryMap[evCopy.ID]
+			s.Unlock()
+			if !ok {
+				// success
+				break
+			}
+
+			if s.retryLimit > -1 && count > s.retryLimit {
+				if logger.V(logger.ErrorLevel, logger.DefaultLogger) {
+					logger.Errorf("Message retry limit reached, discarding: %v %d %d", evCopy.ID, count, s.retryLimit)
+				}
+				s.Lock()
+				delete(s.retryMap, evCopy.ID)
+				s.Unlock()
+				return
+			}
+			s.Channel <- evCopy
+			s.Lock()
+			s.retryMap[evCopy.ID] = count + 1
+			s.Unlock()
+		}
+	}(sub)
+}
+
+func ackFunc(s *subscriber, evCopy events.Event) func() error {
+	return func() error {
+		s.Lock()
+		delete(s.retryMap, evCopy.ID)
+		s.Unlock()
+		return nil
+	}
+}
+
+func nackFunc(s *subscriber, evCopy events.Event) func() error {
+	return func() error {
+		return nil
 	}
 }