Merge pull request #1026 from micro/tun

Network & Tunnel refactor

.github/FUNDING.yml

@@ -0,0 +1,3 @@
+# These are supported funding model platforms
+
+issuehunt: micro/development

.github/ISSUE_TEMPLATE/bug_report.md

@@ -0,0 +1,24 @@
+---
+name: Bug report
+about: For reporting bugs in go-micro
+title: "[BUG]"
+labels: ''
+assignees: ''
+
+---
+
+**Describe the bug**
+
+1. What are you trying to do?
+2. What did you expect to happen?
+3. What happens instead?
+
+**How to reproduce the bug:**
+
+If possible, please include a minimal code snippet here.
+
+**Environment:**
+Go Version: please paste `go version` output here
+```
+please paste `go env` output here
+```

.github/ISSUE_TEMPLATE/feature-request---enhancement.md

@@ -0,0 +1,17 @@
+---
+name: Feature request / Enhancement
+about: If you have a need not served by go-micro
+title: "[FEATURE]"
+labels: ''
+assignees: ''
+
+---
+
+**Is your feature request related to a problem? Please describe.**
+A clear and concise description of what the problem is. Ex. I'm always frustrated when [...]
+
+**Describe the solution you'd like**
+A clear and concise description of what you want to happen.
+
+**Additional context**
+Add any other context or screenshots about the feature request here.

.github/ISSUE_TEMPLATE/question.md

@@ -0,0 +1,14 @@
+---
+name: Question
+about: Ask a question about go-micro
+title: ''
+labels: ''
+assignees: ''
+
+---
+
+Before asking, please check if your question has already been answered: 
+
+1. Check the documentation - https://micro.mu/docs/
+2. Check the examples and plugins - https://github.com/micro/examples & https://github.com/micro/go-plugins
+3. Search existing issues

.golangci.yml

@@ -0,0 +1,26 @@
+run:
+  deadline: 10m
+linters:
+  disable-all: false
+  enable-all: false
+  enable:
+    - megacheck
+    - staticcheck
+    - deadcode
+    - varcheck
+    - gosimple
+    - unused
+    - prealloc
+    - scopelint
+    - gocritic
+    - goimports
+    - unconvert
+    - govet
+    - nakedret
+    - structcheck
+    - gosec
+  disable:
+    - maligned
+    - interfacer
+    - typecheck
+    - dupl

.travis.yml

@@ -2,7 +2,13 @@ language: go
 go:
 - 1.13.x
 env:
-  - GO111MODULE=on
+  - GO111MODULE=on IN_TRAVIS_CI=yes
+before_script:
+  - go install github.com/golangci/golangci-lint/cmd/golangci-lint
+script:
+  - golangci-lint run || true
+  - go test -v -race ./... || true
+  - go test -v ./...
 notifications:
   slack:
     secure: aEvhLbhujaGaKSrOokiG3//PaVHTIrc3fBpoRbCRqfZpyq6WREoapJJhF+tIpWWOwaC9GmChbD6aHo/jMUgwKXVyPSaNjiEL87YzUUpL8B2zslNp1rgfTg/LrzthOx3Q1TYwpaAl3to0fuHUVFX4yMeC2vuThq7WSXgMMxFCtbc=

CNAME

@@ -0,0 +1 @@
+go-micro.dev

Dockerfile

@@ -0,0 +1,13 @@
+FROM golang:1.13-alpine
+
+RUN mkdir /user && \
+    echo 'nobody:x:65534:65534:nobody:/:' > /user/passwd && \
+    echo 'nobody:x:65534:' > /user/group
+
+ENV GO111MODULE=on
+RUN apk --no-cache add make git gcc libtool musl-dev ca-certificates && \
+    rm -rf /var/cache/apk/* /tmp/*
+
+WORKDIR /
+COPY ./go.mod ./go.sum ./
+RUN go mod download && rm go.mod go.sum

README.md

@@ -12,7 +12,7 @@ but everything can be easily swapped out.
 
 Plugins are available at [github.com/micro/go-plugins](https://github.com/micro/go-plugins).
 
-Follow us on [Twitter](https://twitter.com/microhq) or join the [Slack](http://slack.micro.mu/) community.
+Follow us on [Twitter](https://twitter.com/microhq) or join the [Slack](https://micro.mu/slack) community.
 
 ## Features
 
@@ -20,8 +20,7 @@ Go Micro abstracts away the details of distributed systems. Here are the main fe
 
 - **Service Discovery** - Automatic service registration and name resolution. Service discovery is at the core of micro service 
 development. When service A needs to speak to service B it needs the location of that service. The default discovery mechanism is 
-multicast DNS (mdns), a zeroconf system. You can optionally set gossip using the SWIM protocol for p2p networks or consul for a
-resilient cloud-native setup.
+multicast DNS (mdns), a zeroconf system.
 
 - **Load Balancing** - Client side load balancing built on service discovery. Once we have the addresses of any number of instances 
 of a service we now need a way to decide which node to route to. We use random hashed load balancing to provide even distribution 
@@ -45,5 +44,5 @@ are pluggable and allows Go Micro to be runtime agnostic. You can plugin any und
 
 ## Getting Started
 
-See the [docs](https://micro.mu/docs/go-micro.html) for detailed information on the architecture, installation and use of go-micro.
+See the [docs](https://micro.mu/docs/framework.html) for detailed information on the architecture, installation and use of go-micro.
 

_config.yml

@@ -0,0 +1 @@
+theme: jekyll-theme-architect

agent/input/telegram/conn.go

@@ -8,7 +8,7 @@ import (
 	"github.com/forestgiant/sliceutil"
 	"github.com/micro/go-micro/agent/input"
 	"github.com/micro/go-micro/util/log"
-	"gopkg.in/telegram-bot-api.v4"
+	tgbotapi "gopkg.in/telegram-bot-api.v4"
 )
 
 type telegramConn struct {
@@ -44,11 +44,9 @@ func (tc *telegramConn) run() {
 	tc.recv = updates
 	tc.syncCond.Signal()
 
-	for {
-		select {
-		case <-tc.exit:
-			return
-		}
+	select {
+	case <-tc.exit:
+		return
 	}
 }
 

agent/input/telegram/telegram.go

@@ -7,7 +7,7 @@ import (
 
 	"github.com/micro/cli"
 	"github.com/micro/go-micro/agent/input"
-	"gopkg.in/telegram-bot-api.v4"
+	tgbotapi "gopkg.in/telegram-bot-api.v4"
 )
 
 type telegramInput struct {

agent/proto/bot.pb.go

@@ -1,11 +1,13 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
-// source: github.com/micro/go-bot/proto/bot.proto
+// source: github.com/micro/go-micro/agent/proto/bot.proto
 
 package go_micro_bot
 
-import proto "github.com/golang/protobuf/proto"
-import fmt "fmt"
-import math "math"
+import (
+	fmt "fmt"
+	proto "github.com/golang/protobuf/proto"
+	math "math"
+)
 
 // Reference imports to suppress errors if they are not otherwise used.
 var _ = proto.Marshal
@@ -16,7 +18,7 @@ var _ = math.Inf
 // is compatible with the proto package it is being compiled against.
 // A compilation error at this line likely means your copy of the
 // proto package needs to be updated.
-const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package
+const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package
 
 type HelpRequest struct {
 	XXX_NoUnkeyedLiteral struct{} `json:"-"`
@@ -28,16 +30,17 @@ func (m *HelpRequest) Reset()         { *m = HelpRequest{} }
 func (m *HelpRequest) String() string { return proto.CompactTextString(m) }
 func (*HelpRequest) ProtoMessage()    {}
 func (*HelpRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_bot_654832eab83ed4b5, []int{0}
+	return fileDescriptor_018e8d5b14a89d12, []int{0}
 }
+
 func (m *HelpRequest) XXX_Unmarshal(b []byte) error {
 	return xxx_messageInfo_HelpRequest.Unmarshal(m, b)
 }
 func (m *HelpRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
 	return xxx_messageInfo_HelpRequest.Marshal(b, m, deterministic)
 }
-func (dst *HelpRequest) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_HelpRequest.Merge(dst, src)
+func (m *HelpRequest) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_HelpRequest.Merge(m, src)
 }
 func (m *HelpRequest) XXX_Size() int {
 	return xxx_messageInfo_HelpRequest.Size(m)
@@ -60,16 +63,17 @@ func (m *HelpResponse) Reset()         { *m = HelpResponse{} }
 func (m *HelpResponse) String() string { return proto.CompactTextString(m) }
 func (*HelpResponse) ProtoMessage()    {}
 func (*HelpResponse) Descriptor() ([]byte, []int) {
-	return fileDescriptor_bot_654832eab83ed4b5, []int{1}
+	return fileDescriptor_018e8d5b14a89d12, []int{1}
 }
+
 func (m *HelpResponse) XXX_Unmarshal(b []byte) error {
 	return xxx_messageInfo_HelpResponse.Unmarshal(m, b)
 }
 func (m *HelpResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
 	return xxx_messageInfo_HelpResponse.Marshal(b, m, deterministic)
 }
-func (dst *HelpResponse) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_HelpResponse.Merge(dst, src)
+func (m *HelpResponse) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_HelpResponse.Merge(m, src)
 }
 func (m *HelpResponse) XXX_Size() int {
 	return xxx_messageInfo_HelpResponse.Size(m)
@@ -105,16 +109,17 @@ func (m *ExecRequest) Reset()         { *m = ExecRequest{} }
 func (m *ExecRequest) String() string { return proto.CompactTextString(m) }
 func (*ExecRequest) ProtoMessage()    {}
 func (*ExecRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_bot_654832eab83ed4b5, []int{2}
+	return fileDescriptor_018e8d5b14a89d12, []int{2}
 }
+
 func (m *ExecRequest) XXX_Unmarshal(b []byte) error {
 	return xxx_messageInfo_ExecRequest.Unmarshal(m, b)
 }
 func (m *ExecRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
 	return xxx_messageInfo_ExecRequest.Marshal(b, m, deterministic)
 }
-func (dst *ExecRequest) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_ExecRequest.Merge(dst, src)
+func (m *ExecRequest) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_ExecRequest.Merge(m, src)
 }
 func (m *ExecRequest) XXX_Size() int {
 	return xxx_messageInfo_ExecRequest.Size(m)
@@ -144,16 +149,17 @@ func (m *ExecResponse) Reset()         { *m = ExecResponse{} }
 func (m *ExecResponse) String() string { return proto.CompactTextString(m) }
 func (*ExecResponse) ProtoMessage()    {}
 func (*ExecResponse) Descriptor() ([]byte, []int) {
-	return fileDescriptor_bot_654832eab83ed4b5, []int{3}
+	return fileDescriptor_018e8d5b14a89d12, []int{3}
 }
+
 func (m *ExecResponse) XXX_Unmarshal(b []byte) error {
 	return xxx_messageInfo_ExecResponse.Unmarshal(m, b)
 }
 func (m *ExecResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
 	return xxx_messageInfo_ExecResponse.Marshal(b, m, deterministic)
 }
-func (dst *ExecResponse) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_ExecResponse.Merge(dst, src)
+func (m *ExecResponse) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_ExecResponse.Merge(m, src)
 }
 func (m *ExecResponse) XXX_Size() int {
 	return xxx_messageInfo_ExecResponse.Size(m)
@@ -186,25 +192,25 @@ func init() {
 }
 
 func init() {
-	proto.RegisterFile("github.com/micro/go-bot/proto/bot.proto", fileDescriptor_bot_654832eab83ed4b5)
+	proto.RegisterFile("github.com/micro/go-micro/agent/proto/bot.proto", fileDescriptor_018e8d5b14a89d12)
 }
 
-var fileDescriptor_bot_654832eab83ed4b5 = []byte{
+var fileDescriptor_018e8d5b14a89d12 = []byte{
 	// 246 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x6c, 0x90, 0x41, 0x4b, 0xc4, 0x30,
-	0x10, 0x85, 0xb7, 0xba, 0xae, 0xec, 0xb4, 0x5e, 0x82, 0x48, 0xdd, 0x53, 0xcd, 0xc5, 0xbd, 0x98,
-	0x82, 0x5e, 0x05, 0x0f, 0xa2, 0x78, 0xee, 0x3f, 0x68, 0xba, 0x43, 0x2c, 0x6c, 0x3b, 0x35, 0x99,
-	0x82, 0xff, 0xc1, 0x3f, 0x2d, 0x4d, 0x72, 0x08, 0xcb, 0xde, 0xe6, 0x65, 0x86, 0xf7, 0xbe, 0x17,
-	0x78, 0x34, 0x3d, 0x7f, 0xcf, 0x5a, 0x75, 0x34, 0xd4, 0x43, 0xdf, 0x59, 0xaa, 0x0d, 0x3d, 0x69,
-	0xe2, 0x7a, 0xb2, 0xc4, 0x54, 0x6b, 0x62, 0xe5, 0x27, 0x51, 0x18, 0x52, 0xfe, 0x40, 0x69, 0x62,
-	0x79, 0x03, 0xf9, 0x17, 0x1e, 0xa7, 0x06, 0x7f, 0x66, 0x74, 0x2c, 0x3f, 0xa1, 0x08, 0xd2, 0x4d,
-	0x34, 0x3a, 0x14, 0xb7, 0x70, 0x35, 0xbb, 0xd6, 0x60, 0x99, 0x55, 0xd9, 0x7e, 0xdb, 0x04, 0x21,
-	0x2a, 0xc8, 0x0f, 0xe8, 0x3a, 0xdb, 0x4f, 0xdc, 0xd3, 0x58, 0x5e, 0xf8, 0x5d, 0xfa, 0x24, 0x1f,
-	0x20, 0xff, 0xf8, 0xc5, 0x2e, 0xda, 0x0a, 0x01, 0xeb, 0xd6, 0x1a, 0x57, 0x66, 0xd5, 0xe5, 0x7e,
-	0xdb, 0xf8, 0x59, 0xbe, 0x42, 0x11, 0x4e, 0x62, 0xd4, 0x1d, 0x6c, 0x2c, 0xba, 0xf9, 0xc8, 0x3e,
-	0xab, 0x68, 0xa2, 0x5a, 0x10, 0xd0, 0x5a, 0xb2, 0x31, 0x26, 0x88, 0xe7, 0xbf, 0x0c, 0xae, 0xdf,
-	0x69, 0x18, 0xda, 0xf1, 0x20, 0xde, 0x60, 0xbd, 0x40, 0x8b, 0x7b, 0x95, 0x56, 0x53, 0x49, 0xaf,
-	0xdd, 0xee, 0xdc, 0x2a, 0x04, 0xcb, 0xd5, 0x62, 0xb0, 0xa0, 0x9c, 0x1a, 0x24, 0x0d, 0x4e, 0x0d,
-	0x52, 0x72, 0xb9, 0xd2, 0x1b, 0xff, 0xb5, 0x2f, 0xff, 0x01, 0x00, 0x00, 0xff, 0xff, 0xcb, 0x77,
-	0xdf, 0x28, 0x85, 0x01, 0x00, 0x00,
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x6c, 0x50, 0x4d, 0x4b, 0xc4, 0x30,
+	0x10, 0xdd, 0xea, 0xba, 0xb2, 0xd3, 0x7a, 0x09, 0x22, 0x75, 0x4f, 0x35, 0xa7, 0xbd, 0x98, 0x80,
+	0x5e, 0x05, 0x0f, 0xa2, 0x78, 0xee, 0x3f, 0x68, 0xbb, 0x43, 0x2c, 0x6c, 0x3b, 0x35, 0x99, 0x82,
+	0xff, 0xc1, 0x3f, 0x2d, 0x4d, 0x72, 0x08, 0xc5, 0xdb, 0x7b, 0x79, 0xe1, 0x7d, 0x0c, 0x68, 0xd3,
+	0xf3, 0xd7, 0xdc, 0xaa, 0x8e, 0x06, 0x3d, 0xf4, 0x9d, 0x25, 0x6d, 0xe8, 0x31, 0x80, 0xc6, 0xe0,
+	0xc8, 0x7a, 0xb2, 0xc4, 0xa4, 0x5b, 0x62, 0xe5, 0x91, 0x28, 0x0c, 0x29, 0xaf, 0xab, 0x96, 0x58,
+	0xde, 0x40, 0xfe, 0x89, 0xe7, 0xa9, 0xc6, 0xef, 0x19, 0x1d, 0xcb, 0x0f, 0x28, 0x02, 0x75, 0x13,
+	0x8d, 0x0e, 0xc5, 0x2d, 0x5c, 0xcd, 0xae, 0x31, 0x58, 0x66, 0x55, 0x76, 0xdc, 0xd7, 0x81, 0x88,
+	0x0a, 0xf2, 0x13, 0xba, 0xce, 0xf6, 0x13, 0xf7, 0x34, 0x96, 0x17, 0x5e, 0x4b, 0x9f, 0xe4, 0x03,
+	0xe4, 0xef, 0x3f, 0xd8, 0x45, 0x5b, 0x21, 0x60, 0xdb, 0x58, 0xe3, 0xca, 0xac, 0xba, 0x3c, 0xee,
+	0x6b, 0x8f, 0xe5, 0x0b, 0x14, 0xe1, 0x4b, 0x8c, 0xba, 0x83, 0x9d, 0x45, 0x37, 0x9f, 0xd9, 0x67,
+	0x15, 0x75, 0x64, 0x4b, 0x05, 0xb4, 0x96, 0x6c, 0x8c, 0x09, 0xe4, 0xe9, 0x37, 0x83, 0xeb, 0x37,
+	0x1a, 0x86, 0x66, 0x3c, 0x89, 0x57, 0xd8, 0x2e, 0xa5, 0xc5, 0xbd, 0x4a, 0xa7, 0xa9, 0x64, 0xd7,
+	0xe1, 0xf0, 0x9f, 0x14, 0x82, 0xe5, 0x66, 0x31, 0x58, 0xaa, 0xac, 0x0d, 0x92, 0x05, 0x6b, 0x83,
+	0xb4, 0xb9, 0xdc, 0xb4, 0x3b, 0x7f, 0xda, 0xe7, 0xbf, 0x00, 0x00, 0x00, 0xff, 0xff, 0x18, 0xbd,
+	0x39, 0x29, 0x8d, 0x01, 0x00, 0x00,
 }

agent/proto/bot.pb.micro.go

@@ -1,23 +1,13 @@
 // Code generated by protoc-gen-micro. DO NOT EDIT.
-// source: github.com/micro/go-bot/proto/bot.proto
+// source: github.com/micro/go-micro/agent/proto/bot.proto
 
-/*
-Package go_micro_bot is a generated protocol buffer package.
-
-It is generated from these files:
-	github.com/micro/go-bot/proto/bot.proto
-
-It has these top-level messages:
-	HelpRequest
-	HelpResponse
-	ExecRequest
-	ExecResponse
-*/
 package go_micro_bot
 
-import proto "github.com/golang/protobuf/proto"
-import fmt "fmt"
-import math "math"
+import (
+	fmt "fmt"
+	proto "github.com/golang/protobuf/proto"
+	math "math"
+)
 
 import (
 	context "context"
@@ -34,7 +24,7 @@ var _ = math.Inf
 // is compatible with the proto package it is being compiled against.
 // A compilation error at this line likely means your copy of the
 // proto package needs to be updated.
-const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package
+const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package
 
 // Reference imports to suppress errors if they are not otherwise used.
 var _ context.Context
@@ -94,12 +84,12 @@ type CommandHandler interface {
 }
 
 func RegisterCommandHandler(s server.Server, hdlr CommandHandler, opts ...server.HandlerOption) error {
-	type _command interface {
+	type command interface {
 		Help(ctx context.Context, in *HelpRequest, out *HelpResponse) error
 		Exec(ctx context.Context, in *ExecRequest, out *ExecResponse) error
 	}
 	type Command struct {
-		_command
+		command
 	}
 	h := &commandHandler{hdlr}
 	return s.Handle(s.NewHandler(&Command{h}, opts...))

api/handler/api/util.go

@@ -29,16 +29,20 @@ func requestToProto(r *http.Request) (*api.Request, error) {
 
 	ct, _, err := mime.ParseMediaType(r.Header.Get("Content-Type"))
 	if err != nil {
-		ct = "application/x-www-form-urlencoded"
+		ct = "text/plain; charset=UTF-8" //default CT is text/plain
 		r.Header.Set("Content-Type", ct)
 	}
 
-	switch ct {
-	case "application/x-www-form-urlencoded":
-		// expect form vals
-	default:
-		data, _ := ioutil.ReadAll(r.Body)
-		req.Body = string(data)
+	//set the body:
+	if r.Body != nil {
+		switch ct {
+		case "application/x-www-form-urlencoded":
+			// expect form vals in Post data
+		default:
+
+			data, _ := ioutil.ReadAll(r.Body)
+			req.Body = string(data)
+		}
 	}
 
 	// Set X-Forwarded-For if it does not exist

api/handler/api/util_test.go

@@ -8,7 +8,7 @@ import (
 
 func TestRequestToProto(t *testing.T) {
 	testData := []*http.Request{
-		&http.Request{
+		{
 			Method: "GET",
 			Header: http.Header{
 				"Header": []string{"test"},

api/handler/cloudevents/event.go

@@ -32,7 +32,7 @@ import (
 	"unicode"
 
 	"github.com/google/uuid"
-	"gopkg.in/go-playground/validator.v9"
+	validator "gopkg.in/go-playground/validator.v9"
 )
 
 const (

api/handler/event/event.go

@@ -2,6 +2,7 @@
 package event
 
 import (
+	"encoding/json"
 	"fmt"
 	"io/ioutil"
 	"net/http"
@@ -91,12 +92,17 @@ func (e *event) ServeHTTP(w http.ResponseWriter, r *http.Request) {
 	}
 
 	// set body
-	b, err := ioutil.ReadAll(r.Body)
-	if err != nil {
-		http.Error(w, err.Error(), 500)
-		return
+	if r.Method == "GET" {
+		bytes, _ := json.Marshal(r.URL.Query())
+		ev.Data = string(bytes)
+	} else {
+		b, err := ioutil.ReadAll(r.Body)
+		if err != nil {
+			http.Error(w, err.Error(), 500)
+			return
+		}
+		ev.Data = string(b)
 	}
-	ev.Data = string(b)
 
 	// get client
 	c := e.options.Service.Client()

api/handler/http/http_test.go

@@ -27,7 +27,7 @@ func testHttp(t *testing.T, path, service, ns string) {
 	s := &registry.Service{
 		Name: service,
 		Nodes: []*registry.Node{
-			&registry.Node{
+			{
 				Id:      service + "-1",
 				Address: l.Addr().String(),
 			},

api/handler/rpc/rpc_test.go

@@ -7,7 +7,7 @@ import (
 	"testing"
 
 	"github.com/golang/protobuf/proto"
-	"github.com/micro/go-micro/api/proto"
+	go_api "github.com/micro/go-micro/api/proto"
 )
 
 func TestRequestPayloadFromRequest(t *testing.T) {

api/proto/api.pb.go

@@ -3,9 +3,11 @@
 
 package go_api
 
-import proto "github.com/golang/protobuf/proto"
-import fmt "fmt"
-import math "math"
+import (
+	fmt "fmt"
+	proto "github.com/golang/protobuf/proto"
+	math "math"
+)
 
 // Reference imports to suppress errors if they are not otherwise used.
 var _ = proto.Marshal
@@ -16,7 +18,7 @@ var _ = math.Inf
 // is compatible with the proto package it is being compiled against.
 // A compilation error at this line likely means your copy of the
 // proto package needs to be updated.
-const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package
+const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package
 
 type Pair struct {
 	Key                  string   `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"`
@@ -30,16 +32,17 @@ func (m *Pair) Reset()         { *m = Pair{} }
 func (m *Pair) String() string { return proto.CompactTextString(m) }
 func (*Pair) ProtoMessage()    {}
 func (*Pair) Descriptor() ([]byte, []int) {
-	return fileDescriptor_api_17a7876430d97ebd, []int{0}
+	return fileDescriptor_7b6696ef87ec1943, []int{0}
 }
+
 func (m *Pair) XXX_Unmarshal(b []byte) error {
 	return xxx_messageInfo_Pair.Unmarshal(m, b)
 }
 func (m *Pair) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
 	return xxx_messageInfo_Pair.Marshal(b, m, deterministic)
 }
-func (dst *Pair) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_Pair.Merge(dst, src)
+func (m *Pair) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_Pair.Merge(m, src)
 }
 func (m *Pair) XXX_Size() int {
 	return xxx_messageInfo_Pair.Size(m)
@@ -83,16 +86,17 @@ func (m *Request) Reset()         { *m = Request{} }
 func (m *Request) String() string { return proto.CompactTextString(m) }
 func (*Request) ProtoMessage()    {}
 func (*Request) Descriptor() ([]byte, []int) {
-	return fileDescriptor_api_17a7876430d97ebd, []int{1}
+	return fileDescriptor_7b6696ef87ec1943, []int{1}
 }
+
 func (m *Request) XXX_Unmarshal(b []byte) error {
 	return xxx_messageInfo_Request.Unmarshal(m, b)
 }
 func (m *Request) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
 	return xxx_messageInfo_Request.Marshal(b, m, deterministic)
 }
-func (dst *Request) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_Request.Merge(dst, src)
+func (m *Request) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_Request.Merge(m, src)
 }
 func (m *Request) XXX_Size() int {
 	return xxx_messageInfo_Request.Size(m)
@@ -167,16 +171,17 @@ func (m *Response) Reset()         { *m = Response{} }
 func (m *Response) String() string { return proto.CompactTextString(m) }
 func (*Response) ProtoMessage()    {}
 func (*Response) Descriptor() ([]byte, []int) {
-	return fileDescriptor_api_17a7876430d97ebd, []int{2}
+	return fileDescriptor_7b6696ef87ec1943, []int{2}
 }
+
 func (m *Response) XXX_Unmarshal(b []byte) error {
 	return xxx_messageInfo_Response.Unmarshal(m, b)
 }
 func (m *Response) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
 	return xxx_messageInfo_Response.Marshal(b, m, deterministic)
 }
-func (dst *Response) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_Response.Merge(dst, src)
+func (m *Response) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_Response.Merge(m, src)
 }
 func (m *Response) XXX_Size() int {
 	return xxx_messageInfo_Response.Size(m)
@@ -230,16 +235,17 @@ func (m *Event) Reset()         { *m = Event{} }
 func (m *Event) String() string { return proto.CompactTextString(m) }
 func (*Event) ProtoMessage()    {}
 func (*Event) Descriptor() ([]byte, []int) {
-	return fileDescriptor_api_17a7876430d97ebd, []int{3}
+	return fileDescriptor_7b6696ef87ec1943, []int{3}
 }
+
 func (m *Event) XXX_Unmarshal(b []byte) error {
 	return xxx_messageInfo_Event.Unmarshal(m, b)
 }
 func (m *Event) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
 	return xxx_messageInfo_Event.Marshal(b, m, deterministic)
 }
-func (dst *Event) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_Event.Merge(dst, src)
+func (m *Event) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_Event.Merge(m, src)
 }
 func (m *Event) XXX_Size() int {
 	return xxx_messageInfo_Event.Size(m)
@@ -298,35 +304,35 @@ func init() {
 }
 
 func init() {
-	proto.RegisterFile("github.com/micro/go-micro/api/proto/api.proto", fileDescriptor_api_17a7876430d97ebd)
+	proto.RegisterFile("github.com/micro/go-micro/api/proto/api.proto", fileDescriptor_7b6696ef87ec1943)
 }
 
-var fileDescriptor_api_17a7876430d97ebd = []byte{
-	// 410 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x53, 0xc1, 0x6e, 0xd4, 0x30,
-	0x10, 0x55, 0xe2, 0x24, 0x6d, 0x66, 0x11, 0x42, 0x3e, 0x20, 0x53, 0x2a, 0xb4, 0xca, 0x85, 0x15,
-	0x52, 0x13, 0x68, 0x39, 0x20, 0xae, 0xb0, 0x2a, 0xc7, 0xca, 0x7f, 0xe0, 0x6d, 0xac, 0xc4, 0x62,
-	0x13, 0x9b, 0xd8, 0xa9, 0xb4, 0x1f, 0xc7, 0x81, 0xcf, 0xe0, 0x6f, 0x90, 0x27, 0xde, 0xdd, 0xb2,
-	0x5a, 0x2e, 0x74, 0x6f, 0x2f, 0xf6, 0x9b, 0x37, 0x6f, 0xde, 0x38, 0xf0, 0xb6, 0x51, 0xae, 0x1d,
-	0x57, 0xe5, 0xbd, 0xee, 0xaa, 0x4e, 0xdd, 0x0f, 0xba, 0x6a, 0xf4, 0x95, 0x30, 0xaa, 0x32, 0x83,
-	0x76, 0xba, 0x12, 0x46, 0x95, 0x88, 0x68, 0xd6, 0xe8, 0x52, 0x18, 0x55, 0xbc, 0x87, 0xe4, 0x4e,
-	0xa8, 0x81, 0xbe, 0x00, 0xf2, 0x5d, 0x6e, 0x58, 0x34, 0x8f, 0x16, 0x39, 0xf7, 0x90, 0xbe, 0x84,
-	0xec, 0x41, 0xac, 0x47, 0x69, 0x59, 0x3c, 0x27, 0x8b, 0x9c, 0x87, 0xaf, 0xe2, 0x17, 0x81, 0x33,
-	0x2e, 0x7f, 0x8c, 0xd2, 0x3a, 0xcf, 0xe9, 0xa4, 0x6b, 0x75, 0x1d, 0x0a, 0xc3, 0x17, 0xa5, 0x90,
-	0x18, 0xe1, 0x5a, 0x16, 0xe3, 0x29, 0x62, 0x7a, 0x03, 0x59, 0x2b, 0x45, 0x2d, 0x07, 0x46, 0xe6,
-	0x64, 0x31, 0xbb, 0x7e, 0x5d, 0x4e, 0x16, 0xca, 0x20, 0x56, 0x7e, 0xc3, 0xdb, 0x65, 0xef, 0x86,
-	0x0d, 0x0f, 0x54, 0xfa, 0x0e, 0x48, 0x23, 0x1d, 0x4b, 0xb0, 0x82, 0x1d, 0x56, 0xdc, 0x4a, 0x37,
-	0xd1, 0x3d, 0x89, 0x5e, 0x41, 0x62, 0xb4, 0x75, 0x2c, 0x45, 0xf2, 0xab, 0x43, 0xf2, 0x9d, 0xb6,
-	0x81, 0x8d, 0x34, 0xef, 0x71, 0xa5, 0xeb, 0x0d, 0xcb, 0x26, 0x8f, 0x1e, 0xfb, 0x14, 0xc6, 0x61,
-	0xcd, 0xce, 0xa6, 0x14, 0xc6, 0x61, 0x7d, 0x71, 0x0b, 0xb3, 0x47, 0xbe, 0x8e, 0xc4, 0x54, 0x40,
-	0x8a, 0xc1, 0xe0, 0xac, 0xb3, 0xeb, 0x67, 0xdb, 0xb6, 0x3e, 0x55, 0x3e, 0x5d, 0x7d, 0x8e, 0x3f,
-	0x45, 0x17, 0x5f, 0xe1, 0x7c, 0x6b, 0xf7, 0x09, 0x2a, 0x4b, 0xc8, 0x77, 0x73, 0xfc, 0xbf, 0x4c,
-	0xf1, 0x33, 0x82, 0x73, 0x2e, 0xad, 0xd1, 0xbd, 0x95, 0xf4, 0x0d, 0x80, 0x75, 0xc2, 0x8d, 0xf6,
-	0x8b, 0xae, 0x25, 0xaa, 0xa5, 0xfc, 0xd1, 0x09, 0xfd, 0xb8, 0x5b, 0x5c, 0x8c, 0xc9, 0x5e, 0xee,
-	0x93, 0x9d, 0x14, 0x8e, 0x6e, 0x6e, 0x1b, 0x2f, 0xd9, 0xc7, 0x7b, 0xb2, 0x30, 0x8b, 0xdf, 0x11,
-	0xa4, 0xcb, 0x07, 0xd9, 0xe3, 0x16, 0x7b, 0xd1, 0xc9, 0x20, 0x82, 0x98, 0x3e, 0x87, 0x58, 0xd5,
-	0xe1, 0xed, 0xc5, 0xaa, 0xa6, 0x97, 0x90, 0x3b, 0xd5, 0x49, 0xeb, 0x44, 0x67, 0xd0, 0x0f, 0xe1,
-	0xfb, 0x03, 0xfa, 0x61, 0x37, 0x5e, 0xf2, 0xf7, 0xc3, 0xc1, 0x06, 0xff, 0x9a, 0xad, 0x16, 0x4e,
-	0xb0, 0x74, 0x6a, 0xea, 0xf1, 0xc9, 0x66, 0x5b, 0x65, 0xf8, 0x83, 0xde, 0xfc, 0x09, 0x00, 0x00,
-	0xff, 0xff, 0x7a, 0xb4, 0xd4, 0x8f, 0xcb, 0x03, 0x00, 0x00,
+var fileDescriptor_7b6696ef87ec1943 = []byte{
+	// 408 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xac, 0x53, 0x4d, 0x8f, 0xd3, 0x30,
+	0x10, 0x55, 0xe2, 0x24, 0xbb, 0x99, 0x22, 0x84, 0x7c, 0x40, 0x66, 0x59, 0xa1, 0x2a, 0xa7, 0x0a,
+	0xa9, 0x29, 0xec, 0x72, 0x40, 0x5c, 0xa1, 0x5a, 0x8e, 0x2b, 0xff, 0x03, 0x77, 0x63, 0x25, 0x16,
+	0x4d, 0x1c, 0x62, 0xa7, 0x52, 0x7f, 0x1c, 0x07, 0x7e, 0x06, 0xff, 0x06, 0x79, 0xec, 0x7e, 0x50,
+	0x95, 0x0b, 0xf4, 0xf6, 0x62, 0xbf, 0x79, 0xf3, 0xe6, 0x8d, 0x03, 0xf3, 0x5a, 0xd9, 0x66, 0x5c,
+	0x95, 0x4f, 0xba, 0x5d, 0xb4, 0xea, 0x69, 0xd0, 0x8b, 0x5a, 0xcf, 0x3d, 0x10, 0xbd, 0x5a, 0xf4,
+	0x83, 0xb6, 0x88, 0x4a, 0x44, 0x34, 0xab, 0x75, 0x29, 0x7a, 0x55, 0xbc, 0x83, 0xe4, 0x51, 0xa8,
+	0x81, 0xbe, 0x00, 0xf2, 0x4d, 0x6e, 0x59, 0x34, 0x8d, 0x66, 0x39, 0x77, 0x90, 0xbe, 0x84, 0x6c,
+	0x23, 0xd6, 0xa3, 0x34, 0x2c, 0x9e, 0x92, 0x59, 0xce, 0xc3, 0x57, 0xf1, 0x93, 0xc0, 0x15, 0x97,
+	0xdf, 0x47, 0x69, 0xac, 0xe3, 0xb4, 0xd2, 0x36, 0xba, 0x0a, 0x85, 0xe1, 0x8b, 0x52, 0x48, 0x7a,
+	0x61, 0x1b, 0x16, 0xe3, 0x29, 0x62, 0x7a, 0x0f, 0x59, 0x23, 0x45, 0x25, 0x07, 0x46, 0xa6, 0x64,
+	0x36, 0xb9, 0x7b, 0x5d, 0x7a, 0x0b, 0x65, 0x10, 0x2b, 0xbf, 0xe2, 0xed, 0xb2, 0xb3, 0xc3, 0x96,
+	0x07, 0x2a, 0x7d, 0x0b, 0xa4, 0x96, 0x96, 0x25, 0x58, 0xc1, 0x4e, 0x2b, 0x1e, 0xa4, 0xf5, 0x74,
+	0x47, 0xa2, 0x73, 0x48, 0x7a, 0x6d, 0x2c, 0x4b, 0x91, 0xfc, 0xea, 0x94, 0xfc, 0xa8, 0x4d, 0x60,
+	0x23, 0xcd, 0x79, 0x5c, 0xe9, 0x6a, 0xcb, 0x32, 0xef, 0xd1, 0x61, 0x97, 0xc2, 0x38, 0xac, 0xd9,
+	0x95, 0x4f, 0x61, 0x1c, 0xd6, 0x37, 0x0f, 0x30, 0x39, 0xf2, 0x75, 0x26, 0xa6, 0x02, 0x52, 0x0c,
+	0x06, 0x67, 0x9d, 0xdc, 0x3d, 0xdb, 0xb5, 0x75, 0xa9, 0x72, 0x7f, 0xf5, 0x29, 0xfe, 0x18, 0xdd,
+	0x7c, 0x81, 0xeb, 0x9d, 0xdd, 0xff, 0x50, 0x59, 0x42, 0xbe, 0x9f, 0xe3, 0xdf, 0x65, 0x8a, 0x1f,
+	0x11, 0x5c, 0x73, 0x69, 0x7a, 0xdd, 0x19, 0x49, 0xdf, 0x00, 0x18, 0x2b, 0xec, 0x68, 0x3e, 0xeb,
+	0x4a, 0xa2, 0x5a, 0xca, 0x8f, 0x4e, 0xe8, 0x87, 0xfd, 0xe2, 0x62, 0x4c, 0xf6, 0xf6, 0x90, 0xac,
+	0x57, 0x38, 0xbb, 0xb9, 0x5d, 0xbc, 0xe4, 0x10, 0xef, 0xc5, 0xc2, 0x2c, 0x7e, 0x45, 0x90, 0x2e,
+	0x37, 0xb2, 0xc3, 0x2d, 0x76, 0xa2, 0x95, 0x41, 0x04, 0x31, 0x7d, 0x0e, 0xb1, 0xaa, 0xc2, 0xdb,
+	0x8b, 0x55, 0x45, 0x6f, 0x21, 0xb7, 0xaa, 0x95, 0xc6, 0x8a, 0xb6, 0x47, 0x3f, 0x84, 0x1f, 0x0e,
+	0xe8, 0xfb, 0xfd, 0x78, 0xc9, 0x9f, 0x0f, 0x07, 0x1b, 0xfc, 0x6d, 0xb6, 0x4a, 0x58, 0xc1, 0x52,
+	0xdf, 0xd4, 0xe1, 0x8b, 0xcd, 0xb6, 0xca, 0xf0, 0x07, 0xbd, 0xff, 0x1d, 0x00, 0x00, 0xff, 0xff,
+	0x97, 0xf3, 0x59, 0x6e, 0xd1, 0x03, 0x00, 0x00,
 }

api/proto/api.pb.micro.go

@@ -1,23 +1,13 @@
 // Code generated by protoc-gen-micro. DO NOT EDIT.
 // source: github.com/micro/go-micro/api/proto/api.proto
 
-/*
-Package go_api is a generated protocol buffer package.
-
-It is generated from these files:
-	github.com/micro/go-micro/api/proto/api.proto
-
-It has these top-level messages:
-	Pair
-	Request
-	Response
-	Event
-*/
 package go_api
 
-import proto "github.com/golang/protobuf/proto"
-import fmt "fmt"
-import math "math"
+import (
+	fmt "fmt"
+	proto "github.com/golang/protobuf/proto"
+	math "math"
+)
 
 // Reference imports to suppress errors if they are not otherwise used.
 var _ = proto.Marshal
@@ -28,4 +18,4 @@ var _ = math.Inf
 // is compatible with the proto package it is being compiled against.
 // A compilation error at this line likely means your copy of the
 // proto package needs to be updated.
-const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package
+const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package

api/server/acme/acme.go

@@ -0,0 +1,24 @@
+// Package acme abstracts away various ACME libraries
+package acme
+
+import (
+	"errors"
+	"net"
+)
+
+var (
+	// ErrProviderNotImplemented can be returned when attempting to
+	// instantiate an unimplemented provider
+	ErrProviderNotImplemented = errors.New("Provider not implemented")
+)
+
+// Provider is a ACME provider interface
+type Provider interface {
+	NewListener(...string) (net.Listener, error)
+}
+
+// The Let's Encrypt ACME endpoints
+const (
+	LetsEncryptStagingCA    = "https://acme-staging-v02.api.letsencrypt.org/directory"
+	LetsEncryptProductionCA = "https://acme-v02.api.letsencrypt.org/directory"
+)

api/server/acme/autocert/autocert.go

@@ -0,0 +1,23 @@
+// Package autocert is the ACME provider from golang.org/x/crypto/acme/autocert
+// This provider does not take any config.
+package autocert
+
+import (
+	"net"
+
+	"github.com/micro/go-micro/api/server/acme"
+	"golang.org/x/crypto/acme/autocert"
+)
+
+// autoCertACME is the ACME provider from golang.org/x/crypto/acme/autocert
+type autocertProvider struct{}
+
+// NewListener implements acme.Provider
+func (a *autocertProvider) NewListener(ACMEHosts ...string) (net.Listener, error) {
+	return autocert.NewListener(ACMEHosts...), nil
+}
+
+// New returns an autocert acme.Provider
+func New() acme.Provider {
+	return &autocertProvider{}
+}

api/server/acme/autocert/autocert_test.go

@@ -0,0 +1,16 @@
+package autocert
+
+import (
+	"testing"
+)
+
+func TestAutocert(t *testing.T) {
+	l := New()
+	if _, ok := l.(*autocertProvider); !ok {
+		t.Error("New() didn't return an autocertProvider")
+	}
+	// TODO: Travis CI doesn't let us bind :443
+	// if _, err := l.NewListener(); err != nil {
+	// 	t.Error(err.Error())
+	// }
+}

api/server/acme/certmagic/certmagic.go

@@ -0,0 +1,58 @@
+// Package certmagic is the ACME provider from github.com/mholt/certmagic
+package certmagic
+
+import (
+	"log"
+	"math/rand"
+	"net"
+	"time"
+
+	"github.com/mholt/certmagic"
+
+	"github.com/micro/go-micro/api/server/acme"
+)
+
+type certmagicProvider struct {
+	opts acme.Options
+}
+
+func (c *certmagicProvider) NewListener(ACMEHosts ...string) (net.Listener, error) {
+	certmagic.Default.CA = c.opts.CA
+	if c.opts.ChallengeProvider != nil {
+		// Enabling DNS Challenge disables the other challenges
+		certmagic.Default.DNSProvider = c.opts.ChallengeProvider
+	}
+	if c.opts.OnDemand {
+		certmagic.Default.OnDemand = new(certmagic.OnDemandConfig)
+	}
+	if c.opts.Cache != nil {
+		// already validated by new()
+		certmagic.Default.Storage = c.opts.Cache.(certmagic.Storage)
+	}
+	// If multiple instances of the provider are running, inject some
+	// randomness so they don't collide
+	rand.Seed(time.Now().UnixNano())
+	randomDuration := (7 * 24 * time.Hour) + (time.Duration(rand.Intn(504)) * time.Hour)
+	certmagic.Default.RenewDurationBefore = randomDuration
+
+	return certmagic.Listen(ACMEHosts)
+}
+
+// New returns a certmagic provider
+func New(options ...acme.Option) acme.Provider {
+	opts := acme.DefaultOptions()
+
+	for _, o := range options {
+		o(&opts)
+	}
+
+	if opts.Cache != nil {
+		if _, ok := opts.Cache.(certmagic.Storage); !ok {
+			log.Fatal("ACME: cache provided doesn't implement certmagic's Storage interface")
+		}
+	}
+
+	return &certmagicProvider{
+		opts: opts,
+	}
+}

api/server/acme/certmagic/certmagic_test.go

@@ -0,0 +1,224 @@
+package certmagic
+
+import (
+	"net/http"
+	"os"
+	"reflect"
+	"sort"
+	"testing"
+	"time"
+
+	"github.com/go-acme/lego/v3/providers/dns/cloudflare"
+	"github.com/mholt/certmagic"
+	"github.com/micro/go-micro/api/server/acme"
+	cfstore "github.com/micro/go-micro/store/cloudflare"
+	"github.com/micro/go-micro/sync/lock/memory"
+)
+
+func TestCertMagic(t *testing.T) {
+	if len(os.Getenv("IN_TRAVIS_CI")) != 0 {
+		t.Skip("Travis doesn't let us bind :443")
+	}
+	l, err := New().NewListener()
+	if err != nil {
+		t.Fatal(err.Error())
+	}
+	l.Close()
+
+	c := cloudflare.NewDefaultConfig()
+	c.AuthEmail = ""
+	c.AuthKey = ""
+	c.AuthToken = "test"
+	c.ZoneToken = "test"
+
+	p, err := cloudflare.NewDNSProviderConfig(c)
+	if err != nil {
+		t.Fatal(err.Error())
+	}
+
+	l, err = New(acme.AcceptToS(true),
+		acme.CA(acme.LetsEncryptStagingCA),
+		acme.ChallengeProvider(p),
+	).NewListener()
+
+	if err != nil {
+		t.Fatal(err.Error())
+	}
+	l.Close()
+}
+
+func TestStorageImplementation(t *testing.T) {
+	apiToken, accountID := os.Getenv("CF_API_TOKEN"), os.Getenv("CF_ACCOUNT_ID")
+	kvID := os.Getenv("KV_NAMESPACE_ID")
+	if len(apiToken) == 0 || len(accountID) == 0 || len(kvID) == 0 {
+		t.Skip("No Cloudflare API keys available, skipping test")
+	}
+
+	var s certmagic.Storage
+	st := cfstore.NewStore(
+		cfstore.Token(apiToken),
+		cfstore.Account(accountID),
+		cfstore.Namespace(kvID),
+	)
+	s = &storage{
+		lock:  memory.NewLock(),
+		store: st,
+	}
+
+	// Test Lock
+	if err := s.Lock("test"); err != nil {
+		t.Fatal(err)
+	}
+
+	// Test Unlock
+	if err := s.Unlock("test"); err != nil {
+		t.Fatal(err)
+	}
+
+	// Test data
+	testdata := []struct {
+		key   string
+		value []byte
+	}{
+		{key: "/foo/a", value: []byte("lorem")},
+		{key: "/foo/b", value: []byte("ipsum")},
+		{key: "/foo/c", value: []byte("dolor")},
+		{key: "/foo/d", value: []byte("sit")},
+		{key: "/bar/a", value: []byte("amet")},
+		{key: "/bar/b", value: []byte("consectetur")},
+		{key: "/bar/c", value: []byte("adipiscing")},
+		{key: "/bar/d", value: []byte("elit")},
+		{key: "/foo/bar/a", value: []byte("sed")},
+		{key: "/foo/bar/b", value: []byte("do")},
+		{key: "/foo/bar/c", value: []byte("eiusmod")},
+		{key: "/foo/bar/d", value: []byte("tempor")},
+		{key: "/foo/bar/baz/a", value: []byte("incididunt")},
+		{key: "/foo/bar/baz/b", value: []byte("ut")},
+		{key: "/foo/bar/baz/c", value: []byte("labore")},
+		{key: "/foo/bar/baz/d", value: []byte("et")},
+		// a duplicate just in case there's any edge cases
+		{key: "/foo/a", value: []byte("lorem")},
+	}
+
+	// Test Store
+	for _, d := range testdata {
+		if err := s.Store(d.key, d.value); err != nil {
+			t.Fatal(err.Error())
+		}
+	}
+
+	// Test Load
+	for _, d := range testdata {
+		if value, err := s.Load(d.key); err != nil {
+			t.Fatal(err.Error())
+		} else {
+			if !reflect.DeepEqual(value, d.value) {
+				t.Fatalf("Load %s: expected %v, got %v", d.key, d.value, value)
+			}
+		}
+	}
+
+	// Test Exists
+	for _, d := range testdata {
+		if !s.Exists(d.key) {
+			t.Fatalf("%s should exist, but doesn't\n", d.key)
+		}
+	}
+
+	// Test List
+	if list, err := s.List("/", true); err != nil {
+		t.Fatal(err.Error())
+	} else {
+		var expected []string
+		for i, d := range testdata {
+			if i != len(testdata)-1 {
+				// Don't store the intentionally duplicated key
+				expected = append(expected, d.key)
+			}
+		}
+		sort.Strings(expected)
+		sort.Strings(list)
+		if !reflect.DeepEqual(expected, list) {
+			t.Fatalf("List: Expected %v, got %v\n", expected, list)
+		}
+	}
+	if list, err := s.List("/foo", false); err != nil {
+		t.Fatal(err.Error())
+	} else {
+		sort.Strings(list)
+		expected := []string{"/foo/a", "/foo/b", "/foo/bar", "/foo/c", "/foo/d"}
+		if !reflect.DeepEqual(expected, list) {
+			t.Fatalf("List: expected %s, got %s\n", expected, list)
+		}
+	}
+
+	// Test Stat
+	for _, d := range testdata {
+		info, err := s.Stat(d.key)
+		if err != nil {
+			t.Fatal(err.Error())
+		} else {
+			if info.Key != d.key {
+				t.Fatalf("Stat().Key: expected %s, got %s\n", d.key, info.Key)
+			}
+			if info.Size != int64(len(d.value)) {
+				t.Fatalf("Stat().Size: expected %d, got %d\n", len(d.value), info.Size)
+			}
+			if time.Since(info.Modified) > time.Minute {
+				t.Fatalf("Stat().Modified: expected time since last modified to be < 1 minute, got %v\n", time.Since(info.Modified))
+			}
+		}
+
+	}
+
+	// Test Delete
+	for _, d := range testdata {
+		if err := s.Delete(d.key); err != nil {
+			t.Fatal(err.Error())
+		}
+	}
+
+	// New interface doesn't return an error, so call it in case any log.Fatal
+	// happens
+	New(acme.Cache(s))
+}
+
+// Full test with a real zone, with  against LE staging
+func TestE2e(t *testing.T) {
+	apiToken, accountID := os.Getenv("CF_API_TOKEN"), os.Getenv("CF_ACCOUNT_ID")
+	kvID := os.Getenv("KV_NAMESPACE_ID")
+	if len(apiToken) == 0 || len(accountID) == 0 || len(kvID) == 0 {
+		t.Skip("No Cloudflare API keys available, skipping test")
+	}
+
+	testLock := memory.NewLock()
+	testStore := cfstore.NewStore(
+		cfstore.Token(apiToken),
+		cfstore.Account(accountID),
+		cfstore.Namespace(kvID),
+	)
+	testStorage := NewStorage(testLock, testStore)
+
+	conf := cloudflare.NewDefaultConfig()
+	conf.AuthToken = apiToken
+	conf.ZoneToken = apiToken
+	testChallengeProvider, err := cloudflare.NewDNSProviderConfig(conf)
+	if err != nil {
+		t.Fatal(err.Error())
+	}
+
+	testProvider := New(
+		acme.AcceptToS(true),
+		acme.Cache(testStorage),
+		acme.CA(acme.LetsEncryptStagingCA),
+		acme.ChallengeProvider(testChallengeProvider),
+		acme.OnDemand(false),
+	)
+
+	listener, err := testProvider.NewListener("*.micro.mu", "micro.mu")
+	if err != nil {
+		t.Fatal(err.Error())
+	}
+	go http.Serve(listener, http.NotFoundHandler())
+	time.Sleep(10 * time.Minute)
+}

api/server/acme/certmagic/storage.go

@@ -0,0 +1,147 @@
+package certmagic
+
+import (
+	"bytes"
+	"encoding/gob"
+	"errors"
+	"fmt"
+	"path"
+	"strings"
+	"time"
+
+	"github.com/mholt/certmagic"
+	"github.com/micro/go-micro/store"
+	"github.com/micro/go-micro/sync/lock"
+)
+
+// File represents a "File" that will be stored in store.Store - the contents and last modified time
+type File struct {
+	// last modified time
+	LastModified time.Time
+	// Contents
+	Contents []byte
+}
+
+// storage is an implementation of certmagic.Storage using micro's sync.Map and store.Store interfaces.
+// As certmagic storage expects a filesystem (with stat() abilities) we have to implement
+// the bare minimum of metadata.
+type storage struct {
+	lock  lock.Lock
+	store store.Store
+}
+
+func (s *storage) Lock(key string) error {
+	return s.lock.Acquire(key, lock.TTL(10*time.Minute))
+}
+
+func (s *storage) Unlock(key string) error {
+	return s.lock.Release(key)
+}
+
+func (s *storage) Store(key string, value []byte) error {
+	f := File{
+		LastModified: time.Now(),
+		Contents:     value,
+	}
+	buf := &bytes.Buffer{}
+	e := gob.NewEncoder(buf)
+	if err := e.Encode(f); err != nil {
+		return err
+	}
+	r := &store.Record{
+		Key:   key,
+		Value: buf.Bytes(),
+	}
+	return s.store.Write(r)
+}
+
+func (s *storage) Load(key string) ([]byte, error) {
+	if !s.Exists(key) {
+		return nil, certmagic.ErrNotExist(errors.New(key + " doesn't exist"))
+	}
+	records, err := s.store.Read(key)
+	if err != nil {
+		return nil, err
+	}
+	if len(records) != 1 {
+		return nil, fmt.Errorf("ACME Storage: multiple records matched key %s", key)
+	}
+	b := bytes.NewBuffer(records[0].Value)
+	d := gob.NewDecoder(b)
+	var f File
+	err = d.Decode(&f)
+	if err != nil {
+		return nil, err
+	}
+	return f.Contents, nil
+}
+
+func (s *storage) Delete(key string) error {
+	return s.store.Delete(key)
+}
+
+func (s *storage) Exists(key string) bool {
+	if _, err := s.store.Read(key); err != nil {
+		return false
+	}
+	return true
+}
+
+func (s *storage) List(prefix string, recursive bool) ([]string, error) {
+	records, err := s.store.List()
+	if err != nil {
+		return nil, err
+	}
+
+	//nolint:prealloc
+	var results []string
+	for _, r := range records {
+		if strings.HasPrefix(r.Key, prefix) {
+			results = append(results, r.Key)
+		}
+	}
+	if recursive {
+		return results, nil
+	}
+	keysMap := make(map[string]bool)
+	for _, key := range results {
+		dir := strings.Split(strings.TrimPrefix(key, prefix+"/"), "/")
+		keysMap[dir[0]] = true
+	}
+	results = make([]string, 0)
+	for k := range keysMap {
+		results = append(results, path.Join(prefix, k))
+	}
+	return results, nil
+}
+
+func (s *storage) Stat(key string) (certmagic.KeyInfo, error) {
+	records, err := s.store.Read(key)
+	if err != nil {
+		return certmagic.KeyInfo{}, err
+	}
+	if len(records) != 1 {
+		return certmagic.KeyInfo{}, fmt.Errorf("ACME Storage: multiple records matched key %s", key)
+	}
+	b := bytes.NewBuffer(records[0].Value)
+	d := gob.NewDecoder(b)
+	var f File
+	err = d.Decode(&f)
+	if err != nil {
+		return certmagic.KeyInfo{}, err
+	}
+	return certmagic.KeyInfo{
+		Key:        key,
+		Modified:   f.LastModified,
+		Size:       int64(len(f.Contents)),
+		IsTerminal: false,
+	}, nil
+}
+
+// NewStorage returns a certmagic.Storage backed by a go-micro/lock and go-micro/store
+func NewStorage(lock lock.Lock, store store.Store) certmagic.Storage {
+	return &storage{
+		lock:  lock,
+		store: store,
+	}
+}

api/server/acme/options.go

@@ -0,0 +1,73 @@
+package acme
+
+import "github.com/go-acme/lego/v3/challenge"
+
+// Option (or Options) are passed to New() to configure providers
+type Option func(o *Options)
+
+// Options represents various options you can present to ACME providers
+type Options struct {
+	// AcceptTLS must be set to true to indicate that you have read your
+	// provider's terms of service.
+	AcceptToS bool
+	// CA is the CA to use
+	CA string
+	// ChallengeProvider is a go-acme/lego challenge provider. Set this if you
+	// want to use DNS Challenges. Otherwise, tls-alpn-01 will be used
+	ChallengeProvider challenge.Provider
+	// Issue certificates for domains on demand. Otherwise, certs will be
+	// retrieved / issued on start-up.
+	OnDemand bool
+	// Cache is a storage interface. Most ACME libraries have an cache, but
+	// there's no defined interface, so if you consume this option
+	// sanity check it before using.
+	Cache interface{}
+}
+
+// AcceptToS indicates whether you accept your CA's terms of service
+func AcceptToS(b bool) Option {
+	return func(o *Options) {
+		o.AcceptToS = b
+	}
+}
+
+// CA sets the CA of an acme.Options
+func CA(CA string) Option {
+	return func(o *Options) {
+		o.CA = CA
+	}
+}
+
+// ChallengeProvider sets the Challenge provider of an acme.Options
+// if set, it enables the DNS challenge, otherwise tls-alpn-01 will be used.
+func ChallengeProvider(p challenge.Provider) Option {
+	return func(o *Options) {
+		o.ChallengeProvider = p
+	}
+}
+
+// OnDemand enables on-demand certificate issuance. Not recommended for use
+// with the DNS challenge, as the first connection may be very slow.
+func OnDemand(b bool) Option {
+	return func(o *Options) {
+		o.OnDemand = b
+	}
+}
+
+// Cache provides a cache / storage interface to the underlying ACME library
+// as there is no standard, this needs to be validated by the underlying
+// implentation.
+func Cache(c interface{}) Option {
+	return func(o *Options) {
+		o.Cache = c
+	}
+}
+
+// DefaultOptions uses the Let's Encrypt Production CA, with DNS Challenge disabled.
+func DefaultOptions() Options {
+	return Options{
+		AcceptToS: true,
+		CA:        LetsEncryptProductionCA,
+		OnDemand:  true,
+	}
+}

api/server/http/http.go

@@ -11,7 +11,6 @@ import (
 	"github.com/gorilla/handlers"
 	"github.com/micro/go-micro/api/server"
 	"github.com/micro/go-micro/util/log"
-	"golang.org/x/crypto/acme/autocert"
 )
 
 type httpServer struct {
@@ -53,9 +52,9 @@ func (s *httpServer) Start() error {
 	var l net.Listener
 	var err error
 
-	if s.opts.EnableACME {
+	if s.opts.EnableACME && s.opts.ACMEProvider != nil {
 		// should we check the address to make sure its using :443?
-		l = autocert.NewListener(s.opts.ACMEHosts...)
+		l, err = s.opts.ACMEProvider.NewListener(s.opts.ACMEHosts...)
 	} else if s.opts.EnableTLS && s.opts.TLSConfig != nil {
 		l, err = tls.Listen("tcp", s.address, s.opts.TLSConfig)
 	} else {

api/server/options.go

@@ -2,15 +2,24 @@ package server
 
 import (
 	"crypto/tls"
+
+	"github.com/micro/go-micro/api/server/acme"
 )
 
 type Option func(o *Options)
 
 type Options struct {
-	EnableACME bool
-	EnableTLS  bool
-	ACMEHosts  []string
-	TLSConfig  *tls.Config
+	EnableACME   bool
+	ACMEProvider acme.Provider
+	EnableTLS    bool
+	ACMEHosts    []string
+	TLSConfig    *tls.Config
+}
+
+func EnableACME(b bool) Option {
+	return func(o *Options) {
+		o.EnableACME = b
+	}
 }
 
 func ACMEHosts(hosts ...string) Option {
@@ -19,9 +28,9 @@ func ACMEHosts(hosts ...string) Option {
 	}
 }
 
-func EnableACME(b bool) Option {
+func ACMEProvider(p acme.Provider) Option {
 	return func(o *Options) {
-		o.EnableACME = b
+		o.ACMEProvider = p
 	}
 }
 

broker/common_test.go

@@ -7,7 +7,7 @@ import (
 var (
 	// mock data
 	testData = map[string][]*registry.Service{
-		"foo": []*registry.Service{
+		"foo": {
 			{
 				Name:    "foo",
 				Version: "1.0.0",

broker/http_broker.go

@@ -64,6 +64,7 @@ type httpEvent struct {
 
 var (
 	DefaultSubPath   = "/_sub"
+	serviceName      = "go.micro.http.broker"
 	broadcastVersion = "ff.http.broadcast"
 	registerTTL      = time.Minute
 	registerInterval = time.Second * 30
@@ -126,7 +127,7 @@ func newHttpBroker(opts ...Option) Broker {
 	}
 
 	h := &httpBroker{
-		id:          "broker-" + uuid.New().String(),
+		id:          uuid.New().String(),
 		address:     addr,
 		opts:        options,
 		r:           reg,
@@ -236,12 +237,13 @@ func (h *httpBroker) unsubscribe(s *httpSubscriber) error {
 	h.Lock()
 	defer h.Unlock()
 
+	//nolint:prealloc
 	var subscribers []*httpSubscriber
 
 	// look for subscriber
 	for _, sub := range h.subscribers[s.topic] {
 		// deregister and skip forward
-		if sub.id == s.id {
+		if sub == s {
 			_ = h.r.Deregister(sub.svc)
 			continue
 		}
@@ -324,15 +326,22 @@ func (h *httpBroker) ServeHTTP(w http.ResponseWriter, req *http.Request) {
 	p := &httpEvent{m: m, t: topic}
 	id := req.Form.Get("id")
 
+	//nolint:prealloc
+	var subs []Handler
+
 	h.RLock()
 	for _, subscriber := range h.subscribers[topic] {
-		if id == subscriber.id {
-			// sub is sync; crufty rate limiting
-			// so we don't hose the cpu
-			subscriber.fn(p)
+		if id != subscriber.id {
+			continue
 		}
+		subs = append(subs, subscriber.fn)
 	}
 	h.RUnlock()
+
+	// execute the handler
+	for _, fn := range subs {
+		fn(p)
+	}
 }
 
 func (h *httpBroker) Address() string {
@@ -420,7 +429,6 @@ func (h *httpBroker) Connect() error {
 }
 
 func (h *httpBroker) Disconnect() error {
-
 	h.RLock()
 	if !h.running {
 		h.RUnlock()
@@ -467,7 +475,7 @@ func (h *httpBroker) Init(opts ...Option) error {
 	}
 
 	if len(h.id) == 0 {
-		h.id = "broker-" + uuid.New().String()
+		h.id = "go.micro.http.broker-" + uuid.New().String()
 	}
 
 	// get registry
@@ -522,7 +530,7 @@ func (h *httpBroker) Publish(topic string, msg *Message, opts ...PublishOption)
 
 	// now attempt to get the service
 	h.RLock()
-	s, err := h.r.GetService("topic:" + topic)
+	s, err := h.r.GetService(serviceName)
 	if err != nil {
 		h.RUnlock()
 		// ignore error
@@ -555,8 +563,24 @@ func (h *httpBroker) Publish(topic string, msg *Message, opts ...PublishOption)
 
 	srv := func(s []*registry.Service, b []byte) {
 		for _, service := range s {
+			var nodes []*registry.Node
+
+			for _, node := range service.Nodes {
+				// only use nodes tagged with broker http
+				if node.Metadata["broker"] != "http" {
+					continue
+				}
+
+				// look for nodes for the topic
+				if node.Metadata["topic"] != topic {
+					continue
+				}
+
+				nodes = append(nodes, node)
+			}
+
 			// only process if we have nodes
-			if len(service.Nodes) == 0 {
+			if len(nodes) == 0 {
 				continue
 			}
 
@@ -566,7 +590,7 @@ func (h *httpBroker) Publish(topic string, msg *Message, opts ...PublishOption)
 				var success bool
 
 				// publish to all nodes
-				for _, node := range service.Nodes {
+				for _, node := range nodes {
 					// publish async
 					if err := pub(node, topic, b); err == nil {
 						success = true
@@ -579,7 +603,7 @@ func (h *httpBroker) Publish(topic string, msg *Message, opts ...PublishOption)
 				}
 			default:
 				// select node to publish to
-				node := service.Nodes[rand.Int()%len(service.Nodes)]
+				node := nodes[rand.Int()%len(nodes)]
 
 				// publish async to one node
 				if err := pub(node, topic, b); err != nil {
@@ -627,9 +651,6 @@ func (h *httpBroker) Subscribe(topic string, handler Handler, opts ...SubscribeO
 		return nil, err
 	}
 
-	// create unique id
-	id := h.id + "." + uuid.New().String()
-
 	var secure bool
 
 	if h.opts.Secure || h.opts.TLSConfig != nil {
@@ -638,10 +659,12 @@ func (h *httpBroker) Subscribe(topic string, handler Handler, opts ...SubscribeO
 
 	// register service
 	node := &registry.Node{
-		Id:      id,
+		Id:      topic + "-" + h.id,
 		Address: mnet.HostPort(addr, port),
 		Metadata: map[string]string{
 			"secure": fmt.Sprintf("%t", secure),
+			"broker": "http",
+			"topic":  topic,
 		},
 	}
 
@@ -652,7 +675,7 @@ func (h *httpBroker) Subscribe(topic string, handler Handler, opts ...SubscribeO
 	}
 
 	service := &registry.Service{
-		Name:    "topic:" + topic,
+		Name:    serviceName,
 		Version: version,
 		Nodes:   []*registry.Node{node},
 	}
@@ -661,7 +684,7 @@ func (h *httpBroker) Subscribe(topic string, handler Handler, opts ...SubscribeO
 	subscriber := &httpSubscriber{
 		opts:  options,
 		hb:    h,
-		id:    id,
+		id:    node.Id,
 		topic: topic,
 		fn:    handler,
 		svc:   service,

broker/http_broker_test.go

@@ -7,15 +7,13 @@ import (
 
 	glog "github.com/go-log/log"
 	"github.com/google/uuid"
+	"github.com/micro/go-micro/registry"
 	"github.com/micro/go-micro/registry/memory"
 	"github.com/micro/go-micro/util/log"
 )
 
-func newTestRegistry() *memory.Registry {
-	r := memory.NewRegistry()
-	m := r.(*memory.Registry)
-	m.Services = testData
-	return m
+func newTestRegistry() registry.Registry {
+	return memory.NewRegistry(memory.Services(testData))
 }
 
 func sub(be *testing.B, c int) {
@@ -125,7 +123,7 @@ func pub(be *testing.B, c int) {
 
 	for i := 0; i < c; i++ {
 		go func() {
-			for _ = range ch {
+			for range ch {
 				if err := b.Publish(topic, msg); err != nil {
 					be.Fatalf("Unexpected publish error: %v", err)
 				}

broker/nats/nats.go

@@ -70,6 +70,7 @@ func (n *natsBroker) Address() string {
 }
 
 func setAddrs(addrs []string) []string {
+	//nolint:prealloc
 	var cAddrs []string
 	for _, addr := range addrs {
 		if len(addr) == 0 {

broker/nats/nats_test.go

@@ -94,6 +94,5 @@ func TestInitAddrs(t *testing.T) {
 				}
 			}
 		})
-
 	}
 }

broker/service/handler/handler.go

@@ -0,0 +1,66 @@
+package handler
+
+import (
+	"context"
+
+	"github.com/micro/go-micro/broker"
+	pb "github.com/micro/go-micro/broker/service/proto"
+	"github.com/micro/go-micro/errors"
+	"github.com/micro/go-micro/util/log"
+)
+
+type Broker struct {
+	Broker broker.Broker
+}
+
+func (b *Broker) Publish(ctx context.Context, req *pb.PublishRequest, rsp *pb.Empty) error {
+	log.Debugf("Publishing message to %s topic", req.Topic)
+	err := b.Broker.Publish(req.Topic, &broker.Message{
+		Header: req.Message.Header,
+		Body:   req.Message.Body,
+	})
+	log.Debugf("Published message to %s topic", req.Topic)
+	if err != nil {
+		return errors.InternalServerError("go.micro.broker", err.Error())
+	}
+	return nil
+}
+
+func (b *Broker) Subscribe(ctx context.Context, req *pb.SubscribeRequest, stream pb.Broker_SubscribeStream) error {
+	errChan := make(chan error, 1)
+
+	// message handler to stream back messages from broker
+	handler := func(p broker.Event) error {
+		if err := stream.Send(&pb.Message{
+			Header: p.Message().Header,
+			Body:   p.Message().Body,
+		}); err != nil {
+			select {
+			case errChan <- err:
+				return err
+			default:
+				return err
+			}
+		}
+		return nil
+	}
+
+	log.Debugf("Subscribing to %s topic", req.Topic)
+	sub, err := b.Broker.Subscribe(req.Topic, handler, broker.Queue(req.Queue))
+	if err != nil {
+		return errors.InternalServerError("go.micro.broker", err.Error())
+	}
+	defer func() {
+		log.Debugf("Unsubscribing from topic %s", req.Topic)
+		sub.Unsubscribe()
+	}()
+
+	select {
+	case <-ctx.Done():
+		log.Debugf("Context done for subscription to topic %s", req.Topic)
+		return nil
+	case err := <-errChan:
+		log.Debugf("Subscription error for topic %s: %v", req.Topic, err)
+		return err
+	}
+}

broker/service/proto/broker.pb.go

@@ -0,0 +1,229 @@
+// Code generated by protoc-gen-go. DO NOT EDIT.
+// source: micro/go-micro/broker/service/proto/broker.proto
+
+package go_micro_broker
+
+import (
+	fmt "fmt"
+	proto "github.com/golang/protobuf/proto"
+	math "math"
+)
+
+// Reference imports to suppress errors if they are not otherwise used.
+var _ = proto.Marshal
+var _ = fmt.Errorf
+var _ = math.Inf
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the proto package it is being compiled against.
+// A compilation error at this line likely means your copy of the
+// proto package needs to be updated.
+const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package
+
+type Empty struct {
+	XXX_NoUnkeyedLiteral struct{} `json:"-"`
+	XXX_unrecognized     []byte   `json:"-"`
+	XXX_sizecache        int32    `json:"-"`
+}
+
+func (m *Empty) Reset()         { *m = Empty{} }
+func (m *Empty) String() string { return proto.CompactTextString(m) }
+func (*Empty) ProtoMessage()    {}
+func (*Empty) Descriptor() ([]byte, []int) {
+	return fileDescriptor_178fdc60944ff5e5, []int{0}
+}
+
+func (m *Empty) XXX_Unmarshal(b []byte) error {
+	return xxx_messageInfo_Empty.Unmarshal(m, b)
+}
+func (m *Empty) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	return xxx_messageInfo_Empty.Marshal(b, m, deterministic)
+}
+func (m *Empty) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_Empty.Merge(m, src)
+}
+func (m *Empty) XXX_Size() int {
+	return xxx_messageInfo_Empty.Size(m)
+}
+func (m *Empty) XXX_DiscardUnknown() {
+	xxx_messageInfo_Empty.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_Empty proto.InternalMessageInfo
+
+type PublishRequest struct {
+	Topic                string   `protobuf:"bytes,1,opt,name=topic,proto3" json:"topic,omitempty"`
+	Message              *Message `protobuf:"bytes,2,opt,name=message,proto3" json:"message,omitempty"`
+	XXX_NoUnkeyedLiteral struct{} `json:"-"`
+	XXX_unrecognized     []byte   `json:"-"`
+	XXX_sizecache        int32    `json:"-"`
+}
+
+func (m *PublishRequest) Reset()         { *m = PublishRequest{} }
+func (m *PublishRequest) String() string { return proto.CompactTextString(m) }
+func (*PublishRequest) ProtoMessage()    {}
+func (*PublishRequest) Descriptor() ([]byte, []int) {
+	return fileDescriptor_178fdc60944ff5e5, []int{1}
+}
+
+func (m *PublishRequest) XXX_Unmarshal(b []byte) error {
+	return xxx_messageInfo_PublishRequest.Unmarshal(m, b)
+}
+func (m *PublishRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	return xxx_messageInfo_PublishRequest.Marshal(b, m, deterministic)
+}
+func (m *PublishRequest) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_PublishRequest.Merge(m, src)
+}
+func (m *PublishRequest) XXX_Size() int {
+	return xxx_messageInfo_PublishRequest.Size(m)
+}
+func (m *PublishRequest) XXX_DiscardUnknown() {
+	xxx_messageInfo_PublishRequest.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_PublishRequest proto.InternalMessageInfo
+
+func (m *PublishRequest) GetTopic() string {
+	if m != nil {
+		return m.Topic
+	}
+	return ""
+}
+
+func (m *PublishRequest) GetMessage() *Message {
+	if m != nil {
+		return m.Message
+	}
+	return nil
+}
+
+type SubscribeRequest struct {
+	Topic                string   `protobuf:"bytes,1,opt,name=topic,proto3" json:"topic,omitempty"`
+	Queue                string   `protobuf:"bytes,2,opt,name=queue,proto3" json:"queue,omitempty"`
+	XXX_NoUnkeyedLiteral struct{} `json:"-"`
+	XXX_unrecognized     []byte   `json:"-"`
+	XXX_sizecache        int32    `json:"-"`
+}
+
+func (m *SubscribeRequest) Reset()         { *m = SubscribeRequest{} }
+func (m *SubscribeRequest) String() string { return proto.CompactTextString(m) }
+func (*SubscribeRequest) ProtoMessage()    {}
+func (*SubscribeRequest) Descriptor() ([]byte, []int) {
+	return fileDescriptor_178fdc60944ff5e5, []int{2}
+}
+
+func (m *SubscribeRequest) XXX_Unmarshal(b []byte) error {
+	return xxx_messageInfo_SubscribeRequest.Unmarshal(m, b)
+}
+func (m *SubscribeRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	return xxx_messageInfo_SubscribeRequest.Marshal(b, m, deterministic)
+}
+func (m *SubscribeRequest) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_SubscribeRequest.Merge(m, src)
+}
+func (m *SubscribeRequest) XXX_Size() int {
+	return xxx_messageInfo_SubscribeRequest.Size(m)
+}
+func (m *SubscribeRequest) XXX_DiscardUnknown() {
+	xxx_messageInfo_SubscribeRequest.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_SubscribeRequest proto.InternalMessageInfo
+
+func (m *SubscribeRequest) GetTopic() string {
+	if m != nil {
+		return m.Topic
+	}
+	return ""
+}
+
+func (m *SubscribeRequest) GetQueue() string {
+	if m != nil {
+		return m.Queue
+	}
+	return ""
+}
+
+type Message struct {
+	Header               map[string]string `protobuf:"bytes,1,rep,name=header,proto3" json:"header,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
+	Body                 []byte            `protobuf:"bytes,2,opt,name=body,proto3" json:"body,omitempty"`
+	XXX_NoUnkeyedLiteral struct{}          `json:"-"`
+	XXX_unrecognized     []byte            `json:"-"`
+	XXX_sizecache        int32             `json:"-"`
+}
+
+func (m *Message) Reset()         { *m = Message{} }
+func (m *Message) String() string { return proto.CompactTextString(m) }
+func (*Message) ProtoMessage()    {}
+func (*Message) Descriptor() ([]byte, []int) {
+	return fileDescriptor_178fdc60944ff5e5, []int{3}
+}
+
+func (m *Message) XXX_Unmarshal(b []byte) error {
+	return xxx_messageInfo_Message.Unmarshal(m, b)
+}
+func (m *Message) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	return xxx_messageInfo_Message.Marshal(b, m, deterministic)
+}
+func (m *Message) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_Message.Merge(m, src)
+}
+func (m *Message) XXX_Size() int {
+	return xxx_messageInfo_Message.Size(m)
+}
+func (m *Message) XXX_DiscardUnknown() {
+	xxx_messageInfo_Message.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_Message proto.InternalMessageInfo
+
+func (m *Message) GetHeader() map[string]string {
+	if m != nil {
+		return m.Header
+	}
+	return nil
+}
+
+func (m *Message) GetBody() []byte {
+	if m != nil {
+		return m.Body
+	}
+	return nil
+}
+
+func init() {
+	proto.RegisterType((*Empty)(nil), "go.micro.broker.Empty")
+	proto.RegisterType((*PublishRequest)(nil), "go.micro.broker.PublishRequest")
+	proto.RegisterType((*SubscribeRequest)(nil), "go.micro.broker.SubscribeRequest")
+	proto.RegisterType((*Message)(nil), "go.micro.broker.Message")
+	proto.RegisterMapType((map[string]string)(nil), "go.micro.broker.Message.HeaderEntry")
+}
+
+func init() {
+	proto.RegisterFile("micro/go-micro/broker/service/proto/broker.proto", fileDescriptor_178fdc60944ff5e5)
+}
+
+var fileDescriptor_178fdc60944ff5e5 = []byte{
+	// 305 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x7c, 0x51, 0x4d, 0x4f, 0xc2, 0x40,
+	0x14, 0x64, 0x41, 0x68, 0x78, 0x18, 0x25, 0x1b, 0x62, 0x1a, 0x2e, 0x62, 0xe3, 0x81, 0x8b, 0x5b,
+	0x52, 0x2f, 0x6a, 0x8c, 0x07, 0x23, 0x89, 0x07, 0x4d, 0xcc, 0x7a, 0xf3, 0xd6, 0x2d, 0x2f, 0xa5,
+	0x81, 0xba, 0x65, 0xb7, 0x25, 0xe9, 0x1f, 0xf1, 0xe4, 0x8f, 0x35, 0xec, 0x16, 0x3f, 0x68, 0xf0,
+	0x36, 0xf3, 0x76, 0x76, 0xde, 0x64, 0x1e, 0x4c, 0xd2, 0x24, 0x52, 0xd2, 0x8f, 0xe5, 0x85, 0x05,
+	0x42, 0xc9, 0x05, 0x2a, 0x5f, 0xa3, 0x5a, 0x27, 0x11, 0xfa, 0x99, 0x92, 0xf9, 0x76, 0xc8, 0x0c,
+	0xa1, 0xc7, 0xb1, 0x64, 0x46, 0xcb, 0xec, 0xd8, 0x73, 0xa0, 0x3d, 0x4d, 0xb3, 0xbc, 0xf4, 0xde,
+	0xe0, 0xe8, 0xa5, 0x10, 0xcb, 0x44, 0xcf, 0x39, 0xae, 0x0a, 0xd4, 0x39, 0x1d, 0x40, 0x3b, 0x97,
+	0x59, 0x12, 0xb9, 0x64, 0x44, 0xc6, 0x5d, 0x6e, 0x09, 0x0d, 0xc0, 0x49, 0x51, 0xeb, 0x30, 0x46,
+	0xb7, 0x39, 0x22, 0xe3, 0x5e, 0xe0, 0xb2, 0x1d, 0x4f, 0xf6, 0x6c, 0xdf, 0xf9, 0x56, 0xe8, 0xdd,
+	0x41, 0xff, 0xb5, 0x10, 0x3a, 0x52, 0x89, 0xc0, 0xff, 0xdd, 0x07, 0xd0, 0x5e, 0x15, 0x58, 0x58,
+	0xef, 0x2e, 0xb7, 0xc4, 0xfb, 0x20, 0xe0, 0x54, 0xa6, 0xf4, 0x16, 0x3a, 0x73, 0x0c, 0x67, 0xa8,
+	0x5c, 0x32, 0x6a, 0x8d, 0x7b, 0xc1, 0xf9, 0xbe, 0xf5, 0xec, 0xd1, 0xc8, 0xa6, 0xef, 0xb9, 0x2a,
+	0x79, 0xf5, 0x87, 0x52, 0x38, 0x10, 0x72, 0x56, 0x1a, 0xfb, 0x43, 0x6e, 0xf0, 0xf0, 0x1a, 0x7a,
+	0xbf, 0xa4, 0xb4, 0x0f, 0xad, 0x05, 0x96, 0x55, 0xac, 0x0d, 0xdc, 0x84, 0x5a, 0x87, 0xcb, 0x9f,
+	0x50, 0x86, 0xdc, 0x34, 0xaf, 0x48, 0xf0, 0x49, 0xa0, 0x73, 0x6f, 0xb6, 0xd2, 0x07, 0x70, 0xaa,
+	0xfe, 0xe8, 0x69, 0x2d, 0xd2, 0xdf, 0x66, 0x87, 0x27, 0x35, 0x81, 0xbd, 0x41, 0x83, 0x3e, 0x41,
+	0xf7, 0xbb, 0x29, 0x7a, 0x56, 0x93, 0xed, 0xb6, 0x38, 0xdc, 0x5b, 0xbe, 0xd7, 0x98, 0x10, 0xd1,
+	0x31, 0x47, 0xbf, 0xfc, 0x0a, 0x00, 0x00, 0xff, 0xff, 0x60, 0x8c, 0x40, 0xd5, 0x28, 0x02, 0x00,
+	0x00,
+}

broker/service/proto/broker.pb.micro.go

@@ -0,0 +1,173 @@
+// Code generated by protoc-gen-micro. DO NOT EDIT.
+// source: micro/go-micro/broker/service/proto/broker.proto
+
+package go_micro_broker
+
+import (
+	fmt "fmt"
+	proto "github.com/golang/protobuf/proto"
+	math "math"
+)
+
+import (
+	context "context"
+	client "github.com/micro/go-micro/client"
+	server "github.com/micro/go-micro/server"
+)
+
+// Reference imports to suppress errors if they are not otherwise used.
+var _ = proto.Marshal
+var _ = fmt.Errorf
+var _ = math.Inf
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the proto package it is being compiled against.
+// A compilation error at this line likely means your copy of the
+// proto package needs to be updated.
+const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package
+
+// Reference imports to suppress errors if they are not otherwise used.
+var _ context.Context
+var _ client.Option
+var _ server.Option
+
+// Client API for Broker service
+
+type BrokerService interface {
+	Publish(ctx context.Context, in *PublishRequest, opts ...client.CallOption) (*Empty, error)
+	Subscribe(ctx context.Context, in *SubscribeRequest, opts ...client.CallOption) (Broker_SubscribeService, error)
+}
+
+type brokerService struct {
+	c    client.Client
+	name string
+}
+
+func NewBrokerService(name string, c client.Client) BrokerService {
+	if c == nil {
+		c = client.NewClient()
+	}
+	if len(name) == 0 {
+		name = "go.micro.broker"
+	}
+	return &brokerService{
+		c:    c,
+		name: name,
+	}
+}
+
+func (c *brokerService) Publish(ctx context.Context, in *PublishRequest, opts ...client.CallOption) (*Empty, error) {
+	req := c.c.NewRequest(c.name, "Broker.Publish", in)
+	out := new(Empty)
+	err := c.c.Call(ctx, req, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *brokerService) Subscribe(ctx context.Context, in *SubscribeRequest, opts ...client.CallOption) (Broker_SubscribeService, error) {
+	req := c.c.NewRequest(c.name, "Broker.Subscribe", &SubscribeRequest{})
+	stream, err := c.c.Stream(ctx, req, opts...)
+	if err != nil {
+		return nil, err
+	}
+	if err := stream.Send(in); err != nil {
+		return nil, err
+	}
+	return &brokerServiceSubscribe{stream}, nil
+}
+
+type Broker_SubscribeService interface {
+	SendMsg(interface{}) error
+	RecvMsg(interface{}) error
+	Close() error
+	Recv() (*Message, error)
+}
+
+type brokerServiceSubscribe struct {
+	stream client.Stream
+}
+
+func (x *brokerServiceSubscribe) Close() error {
+	return x.stream.Close()
+}
+
+func (x *brokerServiceSubscribe) SendMsg(m interface{}) error {
+	return x.stream.Send(m)
+}
+
+func (x *brokerServiceSubscribe) RecvMsg(m interface{}) error {
+	return x.stream.Recv(m)
+}
+
+func (x *brokerServiceSubscribe) Recv() (*Message, error) {
+	m := new(Message)
+	err := x.stream.Recv(m)
+	if err != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
+// Server API for Broker service
+
+type BrokerHandler interface {
+	Publish(context.Context, *PublishRequest, *Empty) error
+	Subscribe(context.Context, *SubscribeRequest, Broker_SubscribeStream) error
+}
+
+func RegisterBrokerHandler(s server.Server, hdlr BrokerHandler, opts ...server.HandlerOption) error {
+	type broker interface {
+		Publish(ctx context.Context, in *PublishRequest, out *Empty) error
+		Subscribe(ctx context.Context, stream server.Stream) error
+	}
+	type Broker struct {
+		broker
+	}
+	h := &brokerHandler{hdlr}
+	return s.Handle(s.NewHandler(&Broker{h}, opts...))
+}
+
+type brokerHandler struct {
+	BrokerHandler
+}
+
+func (h *brokerHandler) Publish(ctx context.Context, in *PublishRequest, out *Empty) error {
+	return h.BrokerHandler.Publish(ctx, in, out)
+}
+
+func (h *brokerHandler) Subscribe(ctx context.Context, stream server.Stream) error {
+	m := new(SubscribeRequest)
+	if err := stream.Recv(m); err != nil {
+		return err
+	}
+	return h.BrokerHandler.Subscribe(ctx, m, &brokerSubscribeStream{stream})
+}
+
+type Broker_SubscribeStream interface {
+	SendMsg(interface{}) error
+	RecvMsg(interface{}) error
+	Close() error
+	Send(*Message) error
+}
+
+type brokerSubscribeStream struct {
+	stream server.Stream
+}
+
+func (x *brokerSubscribeStream) Close() error {
+	return x.stream.Close()
+}
+
+func (x *brokerSubscribeStream) SendMsg(m interface{}) error {
+	return x.stream.Send(m)
+}
+
+func (x *brokerSubscribeStream) RecvMsg(m interface{}) error {
+	return x.stream.Recv(m)
+}
+
+func (x *brokerSubscribeStream) Send(m *Message) error {
+	return x.stream.Send(m)
+}

broker/service/proto/broker.proto

@@ -0,0 +1,25 @@
+syntax = "proto3";
+
+package go.micro.broker;
+
+service Broker {
+	rpc Publish(PublishRequest) returns (Empty) {};
+	rpc Subscribe(SubscribeRequest) returns (stream Message) {};
+}
+
+message Empty {}
+
+message PublishRequest {
+	string topic = 1;
+	Message message = 2;
+}
+
+message SubscribeRequest {
+	string topic = 1;
+	string queue = 2;
+}
+
+message Message {
+	map<string,string> header = 1;
+	bytes body = 2;
+}

broker/service/service.go

@@ -0,0 +1,132 @@
+// Package service provides the broker service client
+package service
+
+import (
+	"context"
+	"time"
+
+	"github.com/micro/go-micro/broker"
+	pb "github.com/micro/go-micro/broker/service/proto"
+	"github.com/micro/go-micro/client"
+	"github.com/micro/go-micro/util/log"
+)
+
+type serviceBroker struct {
+	Addrs   []string
+	Client  pb.BrokerService
+	options broker.Options
+}
+
+var (
+	DefaultName = "go.micro.broker"
+)
+
+func (b *serviceBroker) Address() string {
+	return b.Addrs[0]
+}
+
+func (b *serviceBroker) Connect() error {
+	return nil
+}
+
+func (b *serviceBroker) Disconnect() error {
+	return nil
+}
+
+func (b *serviceBroker) Init(opts ...broker.Option) error {
+	for _, o := range opts {
+		o(&b.options)
+	}
+	return nil
+}
+
+func (b *serviceBroker) Options() broker.Options {
+	return b.options
+}
+
+func (b *serviceBroker) Publish(topic string, msg *broker.Message, opts ...broker.PublishOption) error {
+	log.Debugf("Publishing to topic %s broker %v", topic, b.Addrs)
+	_, err := b.Client.Publish(context.TODO(), &pb.PublishRequest{
+		Topic: topic,
+		Message: &pb.Message{
+			Header: msg.Header,
+			Body:   msg.Body,
+		},
+	}, client.WithAddress(b.Addrs...))
+	return err
+}
+
+func (b *serviceBroker) Subscribe(topic string, handler broker.Handler, opts ...broker.SubscribeOption) (broker.Subscriber, error) {
+	var options broker.SubscribeOptions
+	for _, o := range opts {
+		o(&options)
+	}
+	log.Debugf("Subscribing to topic %s queue %s broker %v", topic, options.Queue, b.Addrs)
+	stream, err := b.Client.Subscribe(context.TODO(), &pb.SubscribeRequest{
+		Topic: topic,
+		Queue: options.Queue,
+	}, client.WithAddress(b.Addrs...), client.WithRequestTimeout(time.Hour))
+	if err != nil {
+		return nil, err
+	}
+
+	sub := &serviceSub{
+		topic:   topic,
+		queue:   options.Queue,
+		handler: handler,
+		stream:  stream,
+		closed:  make(chan bool),
+		options: options,
+	}
+
+	go func() {
+		for {
+			select {
+			case <-sub.closed:
+				log.Debugf("Unsubscribed from topic %s", topic)
+				return
+			default:
+				// run the subscriber
+				log.Debugf("Streaming from broker %v to topic [%s] queue [%s]", b.Addrs, topic, options.Queue)
+				if err := sub.run(); err != nil {
+					log.Debugf("Resubscribing to topic %s broker %v", topic, b.Addrs)
+					stream, err := b.Client.Subscribe(context.TODO(), &pb.SubscribeRequest{
+						Topic: topic,
+						Queue: options.Queue,
+					}, client.WithAddress(b.Addrs...), client.WithRequestTimeout(time.Hour))
+					if err != nil {
+						log.Debugf("Failed to resubscribe to topic %s: %v", topic, err)
+						time.Sleep(time.Second)
+						continue
+					}
+					// new stream
+					sub.stream = stream
+				}
+			}
+		}
+	}()
+
+	return sub, nil
+}
+
+func (b *serviceBroker) String() string {
+	return "service"
+}
+
+func NewBroker(opts ...broker.Option) broker.Broker {
+	var options broker.Options
+	for _, o := range opts {
+		o(&options)
+	}
+
+	addrs := options.Addrs
+	if len(addrs) == 0 {
+		addrs = []string{"127.0.0.1:8001"}
+	}
+
+	return &serviceBroker{
+		Addrs:   addrs,
+		Client:  pb.NewBrokerService(DefaultName, client.DefaultClient),
+		options: options,
+	}
+}

broker/service/subscriber.go

@@ -0,0 +1,101 @@
+package service
+
+import (
+	"github.com/micro/go-micro/broker"
+	pb "github.com/micro/go-micro/broker/service/proto"
+	"github.com/micro/go-micro/util/log"
+)
+
+type serviceSub struct {
+	topic   string
+	queue   string
+	handler broker.Handler
+	stream  pb.Broker_SubscribeService
+	closed  chan bool
+	options broker.SubscribeOptions
+}
+
+type serviceEvent struct {
+	topic   string
+	message *broker.Message
+}
+
+func (s *serviceEvent) Topic() string {
+	return s.topic
+}
+
+func (s *serviceEvent) Message() *broker.Message {
+	return s.message
+}
+
+func (s *serviceEvent) Ack() error {
+	return nil
+}
+
+func (s *serviceSub) isClosed() bool {
+	select {
+	case <-s.closed:
+		return true
+	default:
+		return false
+	}
+}
+
+func (s *serviceSub) run() error {
+	exit := make(chan bool)
+	go func() {
+		select {
+		case <-exit:
+		case <-s.closed:
+		}
+
+		// close the stream
+		s.stream.Close()
+	}()
+
+	for {
+		// TODO: do not fail silently
+		msg, err := s.stream.Recv()
+		if err != nil {
+			log.Debugf("Streaming error for subcription to topic %s: %v", s.Topic(), err)
+
+			// close the exit channel
+			close(exit)
+
+			// don't return an error if we unsubscribed
+			if s.isClosed() {
+				return nil
+			}
+
+			// return stream error
+			return err
+		}
+
+		// TODO: handle error
+		s.handler(&serviceEvent{
+			topic: s.topic,
+			message: &broker.Message{
+				Header: msg.Header,
+				Body:   msg.Body,
+			},
+		})
+	}
+}
+
+func (s *serviceSub) Options() broker.SubscribeOptions {
+	return s.options
+}
+
+func (s *serviceSub) Topic() string {
+	return s.topic
+}
+
+func (s *serviceSub) Unsubscribe() error {
+	select {
+	case <-s.closed:
+		return nil
+	default:
+		close(s.closed)
+	}
+	return nil
+}

client/common_test.go

@@ -7,7 +7,7 @@ import (
 var (
 	// mock data
 	testData = map[string][]*registry.Service{
-		"foo": []*registry.Service{
+		"foo": {
 			{
 				Name:    "foo",
 				Version: "1.0.0",
@@ -15,10 +15,16 @@ var (
 					{
 						Id:      "foo-1.0.0-123",
 						Address: "localhost:9999",
+						Metadata: map[string]string{
+							"protocol": "mucp",
+						},
 					},
 					{
 						Id:      "foo-1.0.0-321",
 						Address: "localhost:9999",
+						Metadata: map[string]string{
+							"protocol": "mucp",
+						},
 					},
 				},
 			},
@@ -29,6 +35,9 @@ var (
 					{
 						Id:      "foo-1.0.1-321",
 						Address: "localhost:6666",
+						Metadata: map[string]string{
+							"protocol": "mucp",
+						},
 					},
 				},
 			},
@@ -39,6 +48,9 @@ var (
 					{
 						Id:      "foo-1.0.3-345",
 						Address: "localhost:8888",
+						Metadata: map[string]string{
+							"protocol": "mucp",
+						},
 					},
 				},
 			},

client/grpc/codec.go

@@ -11,8 +11,6 @@ import (
 	jsoniter "github.com/json-iterator/go"
 	"github.com/micro/go-micro/codec"
 	"github.com/micro/go-micro/codec/bytes"
-	"github.com/micro/go-micro/codec/jsonrpc"
-	"github.com/micro/go-micro/codec/protorpc"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/encoding"
 )
@@ -36,14 +34,6 @@ var (
 		"application/grpc+bytes":   bytesCodec{},
 	}
 
-	defaultRPCCodecs = map[string]codec.NewCodec{
-		"application/json":         jsonrpc.NewCodec,
-		"application/json-rpc":     jsonrpc.NewCodec,
-		"application/protobuf":     protorpc.NewCodec,
-		"application/proto-rpc":    protorpc.NewCodec,
-		"application/octet-stream": protorpc.NewCodec,
-	}
-
 	json = jsoniter.ConfigCompatibleWithStandardLibrary
 )
 
@@ -182,7 +172,7 @@ func (g *grpcCodec) Write(m *codec.Message, v interface{}) error {
 		return g.s.SendMsg(v)
 	}
 	// write the body using the framing codec
-	return g.s.SendMsg(&bytes.Frame{m.Body})
+	return g.s.SendMsg(&bytes.Frame{Data: m.Body})
 }
 
 func (g *grpcCodec) Close() error {

client/grpc/grpc.go

@@ -13,6 +13,7 @@ import (
 	"github.com/micro/go-micro/client"
 	"github.com/micro/go-micro/client/selector"
 	"github.com/micro/go-micro/codec"
+	raw "github.com/micro/go-micro/codec/bytes"
 	"github.com/micro/go-micro/errors"
 	"github.com/micro/go-micro/metadata"
 	"github.com/micro/go-micro/registry"
@@ -110,12 +111,21 @@ func (g *grpcClient) call(ctx context.Context, node *registry.Node, req client.R
 
 	var grr error
 
-	cc, err := g.pool.getConn(address, grpc.WithDefaultCallOptions(grpc.ForceCodec(cf)),
-		grpc.WithTimeout(opts.DialTimeout), g.secure(),
+	grpcDialOptions := []grpc.DialOption{
+		grpc.WithDefaultCallOptions(grpc.ForceCodec(cf)),
+		grpc.WithTimeout(opts.DialTimeout),
+		g.secure(),
 		grpc.WithDefaultCallOptions(
 			grpc.MaxCallRecvMsgSize(maxRecvMsgSize),
 			grpc.MaxCallSendMsgSize(maxSendMsgSize),
-		))
+		),
+	}
+
+	if opts := g.getGrpcDialOptions(); opts != nil {
+		grpcDialOptions = append(grpcDialOptions, opts...)
+	}
+
+	cc, err := g.pool.getConn(address, grpcDialOptions...)
 	if err != nil {
 		return errors.InternalServerError("go.micro.client", fmt.Sprintf("Error sending request: %v", err))
 	}
@@ -127,7 +137,11 @@ func (g *grpcClient) call(ctx context.Context, node *registry.Node, req client.R
 	ch := make(chan error, 1)
 
 	go func() {
-		err := cc.Invoke(ctx, methodToGRPC(req.Service(), req.Endpoint()), req.Body(), rsp, grpc.CallContentSubtype(cf.Name()))
+		grpcCallOptions := []grpc.CallOption{grpc.CallContentSubtype(cf.Name())}
+		if opts := g.getGrpcCallOptions(); opts != nil {
+			grpcCallOptions = append(grpcCallOptions, opts...)
+		}
+		err := cc.Invoke(ctx, methodToGRPC(req.Service(), req.Endpoint()), req.Body(), rsp, grpcCallOptions...)
 		ch <- microError(err)
 	}()
 
@@ -175,7 +189,16 @@ func (g *grpcClient) stream(ctx context.Context, node *registry.Node, req client
 
 	wc := wrapCodec{cf}
 
-	cc, err := grpc.DialContext(dialCtx, address, grpc.WithDefaultCallOptions(grpc.ForceCodec(wc)), g.secure())
+	grpcDialOptions := []grpc.DialOption{
+		grpc.WithDefaultCallOptions(grpc.ForceCodec(wc)),
+		g.secure(),
+	}
+
+	if opts := g.getGrpcDialOptions(); opts != nil {
+		grpcDialOptions = append(grpcDialOptions, opts...)
+	}
+
+	cc, err := grpc.DialContext(dialCtx, address, grpcDialOptions...)
 	if err != nil {
 		return nil, errors.InternalServerError("go.micro.client", fmt.Sprintf("Error sending request: %v", err))
 	}
@@ -186,7 +209,11 @@ func (g *grpcClient) stream(ctx context.Context, node *registry.Node, req client
 		ServerStreams: true,
 	}
 
-	st, err := cc.NewStream(ctx, desc, methodToGRPC(req.Service(), req.Endpoint()))
+	grpcCallOptions := []grpc.CallOption{}
+	if opts := g.getGrpcCallOptions(); opts != nil {
+		grpcCallOptions = append(grpcCallOptions, opts...)
+	}
+	st, err := cc.NewStream(ctx, desc, methodToGRPC(req.Service(), req.Endpoint()), grpcCallOptions...)
 	if err != nil {
 		return nil, errors.InternalServerError("go.micro.client", fmt.Sprintf("Error creating stream: %v", err))
 	}
@@ -255,16 +282,6 @@ func (g *grpcClient) newGRPCCodec(contentType string) (encoding.Codec, error) {
 	return nil, fmt.Errorf("Unsupported Content-Type: %s", contentType)
 }
 
-func (g *grpcClient) newCodec(contentType string) (codec.NewCodec, error) {
-	if c, ok := g.opts.Codecs[contentType]; ok {
-		return c, nil
-	}
-	if cf, ok := defaultRPCCodecs[contentType]; ok {
-		return cf, nil
-	}
-	return nil, fmt.Errorf("Unsupported Content-Type: %s", contentType)
-}
-
 func (g *grpcClient) Init(opts ...client.Option) error {
 	size := g.opts.PoolSize
 	ttl := g.opts.PoolTTL
@@ -312,7 +329,9 @@ func (g *grpcClient) Call(ctx context.Context, req client.Request, rsp interface
 	d, ok := ctx.Deadline()
 	if !ok {
 		// no deadline so we create a new one
-		ctx, _ = context.WithTimeout(ctx, callOpts.RequestTimeout)
+		var cancel context.CancelFunc
+		ctx, cancel = context.WithTimeout(ctx, callOpts.RequestTimeout)
+		defer cancel()
 	} else {
 		// got a deadline so no need to setup context
 		// but we need to set the timeout we pass along
@@ -484,29 +503,56 @@ func (g *grpcClient) Stream(ctx context.Context, req client.Request, opts ...cli
 }
 
 func (g *grpcClient) Publish(ctx context.Context, p client.Message, opts ...client.PublishOption) error {
+	var options client.PublishOptions
+	for _, o := range opts {
+		o(&options)
+	}
+
 	md, ok := metadata.FromContext(ctx)
 	if !ok {
 		md = make(map[string]string)
 	}
 	md["Content-Type"] = p.ContentType()
+	md["Micro-Topic"] = p.Topic()
 
 	cf, err := g.newGRPCCodec(p.ContentType())
 	if err != nil {
 		return errors.InternalServerError("go.micro.client", err.Error())
 	}
 
-	b, err := cf.Marshal(p.Payload())
-	if err != nil {
-		return errors.InternalServerError("go.micro.client", err.Error())
+	var body []byte
+
+	// passed in raw data
+	if d, ok := p.Payload().(*raw.Frame); ok {
+		body = d.Data
+	} else {
+		// set the body
+		b, err := cf.Marshal(p.Payload())
+		if err != nil {
+			return errors.InternalServerError("go.micro.client", err.Error())
+		}
+		body = b
 	}
 
 	g.once.Do(func() {
 		g.opts.Broker.Connect()
 	})
 
-	return g.opts.Broker.Publish(p.Topic(), &broker.Message{
+	topic := p.Topic()
+
+	// get proxy topic
+	if prx := os.Getenv("MICRO_PROXY"); len(prx) > 0 {
+		options.Exchange = prx
+	}
+
+	// get the exchange
+	if len(options.Exchange) > 0 {
+		topic = options.Exchange
+	}
+
+	return g.opts.Broker.Publish(topic, &broker.Message{
 		Header: md,
-		Body:   b,
+		Body:   body,
 	})
 }
 
@@ -514,6 +560,46 @@ func (g *grpcClient) String() string {
 	return "grpc"
 }
 
+func (g *grpcClient) getGrpcDialOptions() []grpc.DialOption {
+	if g.opts.CallOptions.Context == nil {
+		return nil
+	}
+
+	v := g.opts.CallOptions.Context.Value(grpcDialOptions{})
+
+	if v == nil {
+		return nil
+	}
+
+	opts, ok := v.([]grpc.DialOption)
+
+	if !ok {
+		return nil
+	}
+
+	return opts
+}
+
+func (g *grpcClient) getGrpcCallOptions() []grpc.CallOption {
+	if g.opts.CallOptions.Context == nil {
+		return nil
+	}
+
+	v := g.opts.CallOptions.Context.Value(grpcCallOptions{})
+
+	if v == nil {
+		return nil
+	}
+
+	opts, ok := v.([]grpc.CallOption)
+
+	if !ok {
+		return nil
+	}
+
+	return opts
+}
+
 func newClient(opts ...client.Option) client.Client {
 	options := client.Options{
 		Codecs: make(map[string]codec.NewCodec),

client/grpc/grpc_pool_test.go

@@ -1,11 +1,11 @@
 package grpc
 
 import (
+	"context"
 	"net"
 	"testing"
 	"time"
 
-	"context"
 	"google.golang.org/grpc"
 	pgrpc "google.golang.org/grpc"
 	pb "google.golang.org/grpc/examples/helloworld/helloworld"

client/grpc/grpc_test.go

@@ -42,9 +42,12 @@ func TestGRPCClient(t *testing.T) {
 		Name:    "helloworld",
 		Version: "test",
 		Nodes: []*registry.Node{
-			&registry.Node{
+			{
 				Id:      "test-1",
 				Address: l.Addr().String(),
+				Metadata: map[string]string{
+					"protocol": "grpc",
+				},
 			},
 		},
 	})

client/grpc/options.go

@@ -6,6 +6,7 @@ import (
 	"crypto/tls"
 
 	"github.com/micro/go-micro/client"
+	"google.golang.org/grpc"
 	"google.golang.org/grpc/encoding"
 )
 
@@ -23,6 +24,8 @@ type codecsKey struct{}
 type tlsAuth struct{}
 type maxRecvMsgSizeKey struct{}
 type maxSendMsgSizeKey struct{}
+type grpcDialOptions struct{}
+type grpcCallOptions struct{}
 
 // gRPC Codec to be used to encode/decode requests for a given content type
 func Codec(contentType string, c encoding.Codec) client.Option {
@@ -72,3 +75,27 @@ func MaxSendMsgSize(s int) client.Option {
 		o.Context = context.WithValue(o.Context, maxSendMsgSizeKey{}, s)
 	}
 }
+
+//
+// DialOptions to be used to configure gRPC dial options
+//
+func DialOptions(opts ...grpc.DialOption) client.CallOption {
+	return func(o *client.CallOptions) {
+		if o.Context == nil {
+			o.Context = context.Background()
+		}
+		o.Context = context.WithValue(o.Context, grpcDialOptions{}, opts)
+	}
+}
+
+//
+// CallOptions to be used to configure gRPC call options
+//
+func CallOptions(opts ...grpc.CallOption) client.CallOption {
+	return func(o *client.CallOptions) {
+		if o.Context == nil {
+			o.Context = context.Background()
+		}
+		o.Context = context.WithValue(o.Context, grpcCallOptions{}, opts)
+	}
+}

client/grpc/stream.go

@@ -43,14 +43,12 @@ func (g *grpcStream) Send(msg interface{}) error {
 func (g *grpcStream) Recv(msg interface{}) (err error) {
 	defer g.setError(err)
 	if err = g.stream.RecvMsg(msg); err != nil {
-		if err == io.EOF {
-			// #202 - inconsistent gRPC stream behavior
-			// the only way to tell if the stream is done is when we get a EOF on the Recv
-			// here we should close the underlying gRPC ClientConn
-			closeErr := g.conn.Close()
-			if closeErr != nil {
-				err = closeErr
-			}
+		// #202 - inconsistent gRPC stream behavior
+		// the only way to tell if the stream is done is when we get a EOF on the Recv
+		// here we should close the underlying gRPC ClientConn
+		closeErr := g.conn.Close()
+		if err == io.EOF && closeErr != nil {
+			err = closeErr
 		}
 	}
 	return

client/options.go

@@ -125,6 +125,10 @@ func newOptions(options ...Option) Options {
 		opts.Transport = transport.DefaultTransport
 	}
 
+	if opts.Context == nil {
+		opts.Context = context.Background()
+	}
+
 	return opts
 }
 

client/proto/client.pb.go

@@ -1,13 +1,11 @@
 // Code generated by protoc-gen-go. DO NOT EDIT.
-// source: micro/go-micro/client/proto/client.proto
+// source: github.com/micro/go-micro/client/proto/client.proto
 
 package go_micro_client
 
 import (
-	context "context"
 	fmt "fmt"
 	proto "github.com/golang/protobuf/proto"
-	grpc "google.golang.org/grpc"
 	math "math"
 )
 
@@ -36,7 +34,7 @@ func (m *Request) Reset()         { *m = Request{} }
 func (m *Request) String() string { return proto.CompactTextString(m) }
 func (*Request) ProtoMessage()    {}
 func (*Request) Descriptor() ([]byte, []int) {
-	return fileDescriptor_7d733ae29171347b, []int{0}
+	return fileDescriptor_d418333f021a3308, []int{0}
 }
 
 func (m *Request) XXX_Unmarshal(b []byte) error {
@@ -96,7 +94,7 @@ func (m *Response) Reset()         { *m = Response{} }
 func (m *Response) String() string { return proto.CompactTextString(m) }
 func (*Response) ProtoMessage()    {}
 func (*Response) Descriptor() ([]byte, []int) {
-	return fileDescriptor_7d733ae29171347b, []int{1}
+	return fileDescriptor_d418333f021a3308, []int{1}
 }
 
 func (m *Response) XXX_Unmarshal(b []byte) error {
@@ -137,7 +135,7 @@ func (m *Message) Reset()         { *m = Message{} }
 func (m *Message) String() string { return proto.CompactTextString(m) }
 func (*Message) ProtoMessage()    {}
 func (*Message) Descriptor() ([]byte, []int) {
-	return fileDescriptor_7d733ae29171347b, []int{2}
+	return fileDescriptor_d418333f021a3308, []int{2}
 }
 
 func (m *Message) XXX_Unmarshal(b []byte) error {
@@ -186,203 +184,27 @@ func init() {
 }
 
 func init() {
-	proto.RegisterFile("micro/go-micro/client/proto/client.proto", fileDescriptor_7d733ae29171347b)
+	proto.RegisterFile("github.com/micro/go-micro/client/proto/client.proto", fileDescriptor_d418333f021a3308)
 }
 
-var fileDescriptor_7d733ae29171347b = []byte{
-	// 270 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x91, 0x41, 0x4b, 0xc3, 0x40,
-	0x10, 0x85, 0xbb, 0x6d, 0x4c, 0xea, 0x58, 0x10, 0x06, 0x0f, 0x6b, 0x0e, 0x52, 0x73, 0xca, 0xc5,
-	0x54, 0xf4, 0x2c, 0x1e, 0x72, 0x16, 0x24, 0x8a, 0x57, 0x49, 0xb6, 0x43, 0x5c, 0x48, 0x77, 0xd7,
-	0xec, 0xb6, 0x90, 0x1f, 0xe9, 0x7f, 0x12, 0x36, 0xa9, 0x15, 0x6d, 0x2f, 0xbd, 0xcd, 0x9b, 0x6f,
-	0x79, 0x33, 0xfb, 0x06, 0xd2, 0x95, 0x14, 0xad, 0x5e, 0xd4, 0xfa, 0xa6, 0x2f, 0x44, 0x23, 0x49,
-	0xb9, 0x85, 0x69, 0xb5, 0xdb, 0x8a, 0xcc, 0x0b, 0x3c, 0xaf, 0x75, 0xe6, 0xdf, 0x64, 0x7d, 0x3b,
-	0xd9, 0x40, 0x54, 0xd0, 0xe7, 0x9a, 0xac, 0x43, 0x0e, 0x91, 0xa5, 0x76, 0x23, 0x05, 0x71, 0x36,
-	0x67, 0xe9, 0x69, 0xb1, 0x95, 0x18, 0xc3, 0x94, 0xd4, 0xd2, 0x68, 0xa9, 0x1c, 0x1f, 0x7b, 0xf4,
-	0xa3, 0xf1, 0x1a, 0x66, 0x42, 0x2b, 0x47, 0xca, 0xbd, 0xbb, 0xce, 0x10, 0x9f, 0x78, 0x7e, 0x36,
-	0xf4, 0x5e, 0x3b, 0x43, 0x88, 0x10, 0x54, 0x7a, 0xd9, 0xf1, 0x60, 0xce, 0xd2, 0x59, 0xe1, 0xeb,
-	0xe4, 0x0a, 0xa6, 0x05, 0x59, 0xa3, 0x95, 0xdd, 0x71, 0xf6, 0x8b, 0xbf, 0x41, 0xf4, 0x44, 0xd6,
-	0x96, 0x35, 0xe1, 0x05, 0x9c, 0x38, 0x6d, 0xa4, 0x18, 0xb6, 0xea, 0xc5, 0xbf, 0xb9, 0xe3, 0xc3,
-	0x73, 0x27, 0x3b, 0xdf, 0xbb, 0x2f, 0x06, 0x61, 0xee, 0xbf, 0x8e, 0x0f, 0x10, 0xe4, 0x65, 0xd3,
-	0x20, 0xcf, 0xfe, 0x84, 0x92, 0x0d, 0x89, 0xc4, 0x97, 0x7b, 0x48, 0xbf, 0x73, 0x32, 0xc2, 0x1c,
-	0xc2, 0x17, 0xd7, 0x52, 0xb9, 0x3a, 0xd2, 0x20, 0x65, 0xb7, 0x0c, 0x1f, 0x21, 0x7a, 0x5e, 0x57,
-	0x8d, 0xb4, 0x1f, 0x7b, 0x5c, 0x86, 0x00, 0xe2, 0x83, 0x24, 0x19, 0x55, 0xa1, 0xbf, 0xeb, 0xfd,
-	0x77, 0x00, 0x00, 0x00, 0xff, 0xff, 0x0a, 0x76, 0x1f, 0x51, 0x03, 0x02, 0x00, 0x00,
-}
-
-// Reference imports to suppress errors if they are not otherwise used.
-var _ context.Context
-var _ grpc.ClientConn
-
-// This is a compile-time assertion to ensure that this generated file
-// is compatible with the grpc package it is being compiled against.
-const _ = grpc.SupportPackageIsVersion4
-
-// ClientClient is the client API for Client service.
-//
-// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream.
-type ClientClient interface {
-	// Call allows a single request to be made
-	Call(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error)
-	// Stream is a bidirectional stream
-	Stream(ctx context.Context, opts ...grpc.CallOption) (Client_StreamClient, error)
-	// Publish publishes a message and returns an empty Message
-	Publish(ctx context.Context, in *Message, opts ...grpc.CallOption) (*Message, error)
-}
-
-type clientClient struct {
-	cc *grpc.ClientConn
-}
-
-func NewClientClient(cc *grpc.ClientConn) ClientClient {
-	return &clientClient{cc}
-}
-
-func (c *clientClient) Call(ctx context.Context, in *Request, opts ...grpc.CallOption) (*Response, error) {
-	out := new(Response)
-	err := c.cc.Invoke(ctx, "/go.micro.client.Client/Call", in, out, opts...)
-	if err != nil {
-		return nil, err
-	}
-	return out, nil
-}
-
-func (c *clientClient) Stream(ctx context.Context, opts ...grpc.CallOption) (Client_StreamClient, error) {
-	stream, err := c.cc.NewStream(ctx, &_Client_serviceDesc.Streams[0], "/go.micro.client.Client/Stream", opts...)
-	if err != nil {
-		return nil, err
-	}
-	x := &clientStreamClient{stream}
-	return x, nil
-}
-
-type Client_StreamClient interface {
-	Send(*Request) error
-	Recv() (*Response, error)
-	grpc.ClientStream
-}
-
-type clientStreamClient struct {
-	grpc.ClientStream
-}
-
-func (x *clientStreamClient) Send(m *Request) error {
-	return x.ClientStream.SendMsg(m)
-}
-
-func (x *clientStreamClient) Recv() (*Response, error) {
-	m := new(Response)
-	if err := x.ClientStream.RecvMsg(m); err != nil {
-		return nil, err
-	}
-	return m, nil
-}
-
-func (c *clientClient) Publish(ctx context.Context, in *Message, opts ...grpc.CallOption) (*Message, error) {
-	out := new(Message)
-	err := c.cc.Invoke(ctx, "/go.micro.client.Client/Publish", in, out, opts...)
-	if err != nil {
-		return nil, err
-	}
-	return out, nil
-}
-
-// ClientServer is the server API for Client service.
-type ClientServer interface {
-	// Call allows a single request to be made
-	Call(context.Context, *Request) (*Response, error)
-	// Stream is a bidirectional stream
-	Stream(Client_StreamServer) error
-	// Publish publishes a message and returns an empty Message
-	Publish(context.Context, *Message) (*Message, error)
-}
-
-func RegisterClientServer(s *grpc.Server, srv ClientServer) {
-	s.RegisterService(&_Client_serviceDesc, srv)
-}
-
-func _Client_Call_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
-	in := new(Request)
-	if err := dec(in); err != nil {
-		return nil, err
-	}
-	if interceptor == nil {
-		return srv.(ClientServer).Call(ctx, in)
-	}
-	info := &grpc.UnaryServerInfo{
-		Server:     srv,
-		FullMethod: "/go.micro.client.Client/Call",
-	}
-	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
-		return srv.(ClientServer).Call(ctx, req.(*Request))
-	}
-	return interceptor(ctx, in, info, handler)
-}
-
-func _Client_Stream_Handler(srv interface{}, stream grpc.ServerStream) error {
-	return srv.(ClientServer).Stream(&clientStreamServer{stream})
-}
-
-type Client_StreamServer interface {
-	Send(*Response) error
-	Recv() (*Request, error)
-	grpc.ServerStream
-}
-
-type clientStreamServer struct {
-	grpc.ServerStream
-}
-
-func (x *clientStreamServer) Send(m *Response) error {
-	return x.ServerStream.SendMsg(m)
-}
-
-func (x *clientStreamServer) Recv() (*Request, error) {
-	m := new(Request)
-	if err := x.ServerStream.RecvMsg(m); err != nil {
-		return nil, err
-	}
-	return m, nil
-}
-
-func _Client_Publish_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
-	in := new(Message)
-	if err := dec(in); err != nil {
-		return nil, err
-	}
-	if interceptor == nil {
-		return srv.(ClientServer).Publish(ctx, in)
-	}
-	info := &grpc.UnaryServerInfo{
-		Server:     srv,
-		FullMethod: "/go.micro.client.Client/Publish",
-	}
-	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
-		return srv.(ClientServer).Publish(ctx, req.(*Message))
-	}
-	return interceptor(ctx, in, info, handler)
-}
-
-var _Client_serviceDesc = grpc.ServiceDesc{
-	ServiceName: "go.micro.client.Client",
-	HandlerType: (*ClientServer)(nil),
-	Methods: []grpc.MethodDesc{
-		{
-			MethodName: "Call",
-			Handler:    _Client_Call_Handler,
-		},
-		{
-			MethodName: "Publish",
-			Handler:    _Client_Publish_Handler,
-		},
-	},
-	Streams: []grpc.StreamDesc{
-		{
-			StreamName:    "Stream",
-			Handler:       _Client_Stream_Handler,
-			ServerStreams: true,
-			ClientStreams: true,
-		},
-	},
-	Metadata: "micro/go-micro/client/proto/client.proto",
+var fileDescriptor_d418333f021a3308 = []byte{
+	// 279 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x91, 0x31, 0x4f, 0xfb, 0x30,
+	0x10, 0xc5, 0xeb, 0xb6, 0xff, 0xa4, 0xff, 0xa3, 0x12, 0x92, 0xc5, 0x60, 0x32, 0xa0, 0x92, 0x29,
+	0x0b, 0x0e, 0xa2, 0x33, 0x62, 0xc8, 0x8c, 0x84, 0x02, 0x62, 0x45, 0x89, 0x7b, 0x4a, 0x2d, 0x25,
+	0xb6, 0x89, 0x9d, 0x4a, 0xf9, 0x90, 0x7c, 0x27, 0x24, 0x27, 0xa5, 0x08, 0xda, 0x85, 0xed, 0xde,
+	0xfd, 0xac, 0x77, 0xe7, 0x77, 0xb0, 0xae, 0xa4, 0xdb, 0x76, 0x25, 0x17, 0xba, 0x49, 0x1b, 0x29,
+	0x5a, 0x9d, 0x56, 0xfa, 0x66, 0x28, 0x44, 0x2d, 0x51, 0xb9, 0xd4, 0xb4, 0xda, 0xed, 0x05, 0xf7,
+	0x82, 0x9e, 0x57, 0x9a, 0xfb, 0x37, 0x7c, 0x68, 0xc7, 0x3b, 0x08, 0x73, 0x7c, 0xef, 0xd0, 0x3a,
+	0xca, 0x20, 0xb4, 0xd8, 0xee, 0xa4, 0x40, 0x46, 0x56, 0x24, 0xf9, 0x9f, 0xef, 0x25, 0x8d, 0x60,
+	0x81, 0x6a, 0x63, 0xb4, 0x54, 0x8e, 0x4d, 0x3d, 0xfa, 0xd2, 0xf4, 0x1a, 0x96, 0x42, 0x2b, 0x87,
+	0xca, 0xbd, 0xb9, 0xde, 0x20, 0x9b, 0x79, 0x7e, 0x36, 0xf6, 0x5e, 0x7a, 0x83, 0x94, 0xc2, 0xbc,
+	0xd4, 0x9b, 0x9e, 0xcd, 0x57, 0x24, 0x59, 0xe6, 0xbe, 0x8e, 0xaf, 0x60, 0x91, 0xa3, 0x35, 0x5a,
+	0xd9, 0x03, 0x27, 0xdf, 0xf8, 0x2b, 0x84, 0x8f, 0x68, 0x6d, 0x51, 0x21, 0xbd, 0x80, 0x7f, 0x4e,
+	0x1b, 0x29, 0xc6, 0xad, 0x06, 0xf1, 0x6b, 0xee, 0xf4, 0xf4, 0xdc, 0xd9, 0xc1, 0xf7, 0xee, 0x83,
+	0x40, 0x90, 0xf9, 0xaf, 0xd3, 0x7b, 0x98, 0x67, 0x45, 0x5d, 0x53, 0xc6, 0x7f, 0x84, 0xc2, 0xc7,
+	0x44, 0xa2, 0xcb, 0x23, 0x64, 0xd8, 0x39, 0x9e, 0xd0, 0x0c, 0x82, 0x67, 0xd7, 0x62, 0xd1, 0xfc,
+	0xd1, 0x20, 0x21, 0xb7, 0x84, 0x3e, 0x40, 0xf8, 0xd4, 0x95, 0xb5, 0xb4, 0xdb, 0x23, 0x2e, 0x63,
+	0x00, 0xd1, 0x49, 0x12, 0x4f, 0xca, 0xc0, 0xdf, 0x75, 0xfd, 0x19, 0x00, 0x00, 0xff, 0xff, 0xb6,
+	0x4d, 0x6e, 0xd5, 0x0e, 0x02, 0x00, 0x00,
 }

client/proto/client.pb.micro.go

@@ -1,5 +1,5 @@
 // Code generated by protoc-gen-micro. DO NOT EDIT.
-// source: micro/go-micro/client/proto/client.proto
+// source: github.com/micro/go-micro/client/proto/client.proto
 
 package go_micro_client
 

client/rpc_client.go

@@ -13,6 +13,7 @@ import (
 	"github.com/micro/go-micro/client/pool"
 	"github.com/micro/go-micro/client/selector"
 	"github.com/micro/go-micro/codec"
+	raw "github.com/micro/go-micro/codec/bytes"
 	"github.com/micro/go-micro/errors"
 	"github.com/micro/go-micro/metadata"
 	"github.com/micro/go-micro/registry"
@@ -161,7 +162,6 @@ func (r *rpcClient) call(ctx context.Context, node *registry.Node, req Request,
 
 	select {
 	case err := <-ch:
-		grr = err
 		return err
 	case <-ctx.Done():
 		grr = errors.Timeout("go.micro.client", fmt.Sprintf("%v", ctx.Err()))
@@ -316,6 +316,22 @@ func (r *rpcClient) Options() Options {
 	return r.opts
 }
 
+// hasProxy checks if we have proxy set in the environment
+func (r *rpcClient) hasProxy() bool {
+	// get proxy
+	if prx := os.Getenv("MICRO_PROXY"); len(prx) > 0 {
+		return true
+	}
+
+	// get proxy address
+	if prx := os.Getenv("MICRO_PROXY_ADDRESS"); len(prx) > 0 {
+		return true
+	}
+
+	return false
+}
+
+// next returns an iterator for the next nodes to call
 func (r *rpcClient) next(request Request, opts CallOptions) (selector.Next, error) {
 	service := request.Service()
 
@@ -377,7 +393,9 @@ func (r *rpcClient) Call(ctx context.Context, request Request, response interfac
 	d, ok := ctx.Deadline()
 	if !ok {
 		// no deadline so we create a new one
-		ctx, _ = context.WithTimeout(ctx, callOpts.RequestTimeout)
+		var cancel context.CancelFunc
+		ctx, cancel = context.WithTimeout(ctx, callOpts.RequestTimeout)
+		defer cancel()
 	} else {
 		// got a deadline so no need to setup context
 		// but we need to set the timeout we pass along
@@ -429,10 +447,18 @@ func (r *rpcClient) Call(ctx context.Context, request Request, response interfac
 		return err
 	}
 
-	ch := make(chan error, callOpts.Retries+1)
+	// get the retries
+	retries := callOpts.Retries
+
+	// disable retries when using a proxy
+	if r.hasProxy() {
+		retries = 0
+	}
+
+	ch := make(chan error, retries+1)
 	var gerr error
 
-	for i := 0; i <= callOpts.Retries; i++ {
+	for i := 0; i <= retries; i++ {
 		go func(i int) {
 			ch <- call(i)
 		}(i)
@@ -512,10 +538,18 @@ func (r *rpcClient) Stream(ctx context.Context, request Request, opts ...CallOpt
 		err    error
 	}
 
-	ch := make(chan response, callOpts.Retries+1)
+	// get the retries
+	retries := callOpts.Retries
+
+	// disable retries when using a proxy
+	if r.hasProxy() {
+		retries = 0
+	}
+
+	ch := make(chan response, retries+1)
 	var grr error
 
-	for i := 0; i <= callOpts.Retries; i++ {
+	for i := 0; i <= retries; i++ {
 		go func(i int) {
 			s, err := call(i)
 			ch <- response{s, err}
@@ -583,26 +617,37 @@ func (r *rpcClient) Publish(ctx context.Context, msg Message, opts ...PublishOpt
 		return errors.InternalServerError("go.micro.client", err.Error())
 	}
 
-	// new buffer
-	b := buf.New(nil)
+	var body []byte
 
-	if err := cf(b).Write(&codec.Message{
-		Target: topic,
-		Type:   codec.Event,
-		Header: map[string]string{
-			"Micro-Id":    id,
-			"Micro-Topic": msg.Topic(),
-		},
-	}, msg.Payload()); err != nil {
-		return errors.InternalServerError("go.micro.client", err.Error())
+	// passed in raw data
+	if d, ok := msg.Payload().(*raw.Frame); ok {
+		body = d.Data
+	} else {
+		// new buffer
+		b := buf.New(nil)
+
+		if err := cf(b).Write(&codec.Message{
+			Target: topic,
+			Type:   codec.Event,
+			Header: map[string]string{
+				"Micro-Id":    id,
+				"Micro-Topic": msg.Topic(),
+			},
+		}, msg.Payload()); err != nil {
+			return errors.InternalServerError("go.micro.client", err.Error())
+		}
+
+		// set the body
+		body = b.Bytes()
 	}
+
 	r.once.Do(func() {
 		r.opts.Broker.Connect()
 	})
 
 	return r.opts.Broker.Publish(topic, &broker.Message{
 		Header: md,
-		Body:   b.Bytes(),
+		Body:   body,
 	})
 }
 

client/rpc_client_test.go

@@ -12,10 +12,7 @@ import (
 )
 
 func newTestRegistry() registry.Registry {
-	r := memory.NewRegistry()
-	reg := r.(*memory.Registry)
-	reg.Services = testData
-	return reg
+	return memory.NewRegistry(memory.Services(testData))
 }
 
 func TestCallAddress(t *testing.T) {
@@ -143,9 +140,12 @@ func TestCallWrapper(t *testing.T) {
 		Name:    service,
 		Version: "latest",
 		Nodes: []*registry.Node{
-			&registry.Node{
+			{
 				Id:      id,
 				Address: address,
+				Metadata: map[string]string{
+					"protocol": "mucp",
+				},
 			},
 		},
 	})

client/rpc_codec.go

@@ -88,32 +88,24 @@ func (rwc *readWriteCloser) Close() error {
 }
 
 func getHeaders(m *codec.Message) {
-	get := func(hdr string) string {
-		if hd := m.Header[hdr]; len(hd) > 0 {
-			return hd
+	set := func(v, hdr string) string {
+		if len(v) > 0 {
+			return v
 		}
-		// old
-		return m.Header["X-"+hdr]
+		return m.Header[hdr]
 	}
 
 	// check error in header
-	if len(m.Error) == 0 {
-		m.Error = get("Micro-Error")
-	}
+	m.Error = set(m.Error, "Micro-Error")
 
 	// check endpoint in header
-	if len(m.Endpoint) == 0 {
-		m.Endpoint = get("Micro-Endpoint")
-	}
+	m.Endpoint = set(m.Endpoint, "Micro-Endpoint")
 
 	// check method in header
-	if len(m.Method) == 0 {
-		m.Method = get("Micro-Method")
-	}
+	m.Method = set(m.Method, "Micro-Method")
 
-	if len(m.Id) == 0 {
-		m.Id = get("Micro-Id")
-	}
+	// set the request id
+	m.Id = set(m.Id, "Micro-Id")
 }
 
 func setHeaders(m *codec.Message, stream string) {
@@ -122,7 +114,6 @@ func setHeaders(m *codec.Message, stream string) {
 			return
 		}
 		m.Header[hdr] = v
-		m.Header["X-"+hdr] = v
 	}
 
 	set("Micro-Id", m.Id)
@@ -145,6 +136,11 @@ func setupProtocol(msg *transport.Message, node *registry.Node) codec.NewCodec {
 		return nil
 	}
 
+	// processing topic publishing
+	if len(msg.Header["Micro-Topic"]) > 0 {
+		return nil
+	}
+
 	// no protocol use old codecs
 	switch msg.Header["Content-Type"] {
 	case "application/json":

client/selector/common_test.go

@@ -7,7 +7,7 @@ import (
 var (
 	// mock data
 	testData = map[string][]*registry.Service{
-		"foo": []*registry.Service{
+		"foo": {
 			{
 				Name:    "foo",
 				Version: "1.0.0",

client/selector/default_test.go

@@ -9,9 +9,7 @@ import (
 func TestRegistrySelector(t *testing.T) {
 	counts := map[string]int{}
 
-	r := memory.NewRegistry()
-	rg := r.(*memory.Registry)
-	rg.Services = testData
+	r := memory.NewRegistry(memory.Services(testData))
 	cache := NewSelector(Registry(r))
 
 	next, err := cache.Select("foo")

client/selector/dns/dns.go

@@ -63,7 +63,7 @@ func (d *dnsSelector) Select(service string, opts ...selector.SelectOption) (sel
 		}
 	}
 
-	var nodes []*registry.Node
+	nodes := make([]*registry.Node, 0, len(srv))
 	for _, node := range srv {
 		nodes = append(nodes, &registry.Node{
 			Id:      node.Target,
@@ -72,7 +72,7 @@ func (d *dnsSelector) Select(service string, opts ...selector.SelectOption) (sel
 	}
 
 	services := []*registry.Service{
-		&registry.Service{
+		{
 			Name:  service,
 			Nodes: nodes,
 		},
@@ -99,13 +99,9 @@ func (d *dnsSelector) Select(service string, opts ...selector.SelectOption) (sel
 	return sopts.Strategy(services), nil
 }
 
-func (d *dnsSelector) Mark(service string, node *registry.Node, err error) {
-	return
-}
+func (d *dnsSelector) Mark(service string, node *registry.Node, err error) {}
 
-func (d *dnsSelector) Reset(service string) {
-	return
-}
+func (d *dnsSelector) Reset(service string) {}
 
 func (d *dnsSelector) Close() error {
 	return nil

client/selector/filter_test.go

@@ -14,20 +14,20 @@ func TestFilterEndpoint(t *testing.T) {
 	}{
 		{
 			services: []*registry.Service{
-				&registry.Service{
+				{
 					Name:    "test",
 					Version: "1.0.0",
 					Endpoints: []*registry.Endpoint{
-						&registry.Endpoint{
+						{
 							Name: "Foo.Bar",
 						},
 					},
 				},
-				&registry.Service{
+				{
 					Name:    "test",
 					Version: "1.1.0",
 					Endpoints: []*registry.Endpoint{
-						&registry.Endpoint{
+						{
 							Name: "Baz.Bar",
 						},
 					},
@@ -38,20 +38,20 @@ func TestFilterEndpoint(t *testing.T) {
 		},
 		{
 			services: []*registry.Service{
-				&registry.Service{
+				{
 					Name:    "test",
 					Version: "1.0.0",
 					Endpoints: []*registry.Endpoint{
-						&registry.Endpoint{
+						{
 							Name: "Foo.Bar",
 						},
 					},
 				},
-				&registry.Service{
+				{
 					Name:    "test",
 					Version: "1.1.0",
 					Endpoints: []*registry.Endpoint{
-						&registry.Endpoint{
+						{
 							Name: "Foo.Bar",
 						},
 					},
@@ -95,11 +95,11 @@ func TestFilterLabel(t *testing.T) {
 	}{
 		{
 			services: []*registry.Service{
-				&registry.Service{
+				{
 					Name:    "test",
 					Version: "1.0.0",
 					Nodes: []*registry.Node{
-						&registry.Node{
+						{
 							Id:      "test-1",
 							Address: "localhost",
 							Metadata: map[string]string{
@@ -108,11 +108,11 @@ func TestFilterLabel(t *testing.T) {
 						},
 					},
 				},
-				&registry.Service{
+				{
 					Name:    "test",
 					Version: "1.1.0",
 					Nodes: []*registry.Node{
-						&registry.Node{
+						{
 							Id:      "test-2",
 							Address: "localhost",
 							Metadata: map[string]string{
@@ -127,21 +127,21 @@ func TestFilterLabel(t *testing.T) {
 		},
 		{
 			services: []*registry.Service{
-				&registry.Service{
+				{
 					Name:    "test",
 					Version: "1.0.0",
 					Nodes: []*registry.Node{
-						&registry.Node{
+						{
 							Id:      "test-1",
 							Address: "localhost",
 						},
 					},
 				},
-				&registry.Service{
+				{
 					Name:    "test",
 					Version: "1.1.0",
 					Nodes: []*registry.Node{
-						&registry.Node{
+						{
 							Id:      "test-2",
 							Address: "localhost",
 						},
@@ -187,11 +187,11 @@ func TestFilterVersion(t *testing.T) {
 	}{
 		{
 			services: []*registry.Service{
-				&registry.Service{
+				{
 					Name:    "test",
 					Version: "1.0.0",
 				},
-				&registry.Service{
+				{
 					Name:    "test",
 					Version: "1.1.0",
 				},
@@ -201,11 +201,11 @@ func TestFilterVersion(t *testing.T) {
 		},
 		{
 			services: []*registry.Service{
-				&registry.Service{
+				{
 					Name:    "test",
 					Version: "1.0.0",
 				},
-				&registry.Service{
+				{
 					Name:    "test",
 					Version: "1.1.0",
 				},

client/selector/router/router.go

@@ -43,9 +43,9 @@ type routerKey struct{}
 func (r *routerSelector) getRoutes(service string) ([]router.Route, error) {
 	if !r.remote {
 		// lookup router for routes for the service
-		return r.r.Lookup(router.NewQuery(
+		return r.r.Lookup(
 			router.QueryService(service),
-		))
+		)
 	}
 
 	// lookup the remote router
@@ -101,7 +101,7 @@ func (r *routerSelector) getRoutes(service string) ([]router.Route, error) {
 		return nil, selector.ErrNoneAvailable
 	}
 
-	var routes []router.Route
+	routes := make([]router.Route, 0, len(pbRoutes.Routes))
 
 	// convert from pb to []*router.Route
 	for _, r := range pbRoutes.Routes {
@@ -111,7 +111,7 @@ func (r *routerSelector) getRoutes(service string) ([]router.Route, error) {
 			Gateway: r.Gateway,
 			Network: r.Network,
 			Link:    r.Link,
-			Metric:  int(r.Metric),
+			Metric:  r.Metric,
 		})
 	}
 
@@ -176,12 +176,10 @@ func (r *routerSelector) Select(service string, opts ...selector.SelectOption) (
 
 func (r *routerSelector) Mark(service string, node *registry.Node, err error) {
 	// TODO: pass back metrics or information to the router
-	return
 }
 
 func (r *routerSelector) Reset(service string) {
 	// TODO: reset the metrics or information at the router
-	return
 }
 
 func (r *routerSelector) Close() error {

client/selector/selector.go

@@ -3,6 +3,7 @@ package selector
 
 import (
 	"errors"
+
 	"github.com/micro/go-micro/registry"
 )
 

client/selector/strategy.go

@@ -14,7 +14,7 @@ func init() {
 
 // Random is a random strategy algorithm for node selection
 func Random(services []*registry.Service) Next {
-	var nodes []*registry.Node
+	nodes := make([]*registry.Node, 0, len(services))
 
 	for _, service := range services {
 		nodes = append(nodes, service.Nodes...)
@@ -32,7 +32,7 @@ func Random(services []*registry.Service) Next {
 
 // RoundRobin is a roundrobin strategy algorithm for node selection
 func RoundRobin(services []*registry.Service) Next {
-	var nodes []*registry.Node
+	nodes := make([]*registry.Node, 0, len(services))
 
 	for _, service := range services {
 		nodes = append(nodes, service.Nodes...)

client/selector/strategy_test.go

@@ -8,29 +8,29 @@ import (
 
 func TestStrategies(t *testing.T) {
 	testData := []*registry.Service{
-		&registry.Service{
+		{
 			Name:    "test1",
 			Version: "latest",
 			Nodes: []*registry.Node{
-				&registry.Node{
+				{
 					Id:      "test1-1",
 					Address: "10.0.0.1:1001",
 				},
-				&registry.Node{
+				{
 					Id:      "test1-2",
 					Address: "10.0.0.2:1002",
 				},
 			},
 		},
-		&registry.Service{
+		{
 			Name:    "test1",
 			Version: "default",
 			Nodes: []*registry.Node{
-				&registry.Node{
+				{
 					Id:      "test1-3",
 					Address: "10.0.0.3:1003",
 				},
-				&registry.Node{
+				{
 					Id:      "test1-4",
 					Address: "10.0.0.4:1004",
 				},

codec/bytes/bytes.go

@@ -29,12 +29,10 @@ func (c *Codec) ReadBody(b interface{}) error {
 		return err
 	}
 
-	switch b.(type) {
+	switch v := b.(type) {
 	case *[]byte:
-		v := b.(*[]byte)
 		*v = buf
 	case *Frame:
-		v := b.(*Frame)
 		v.Data = buf
 	default:
 		return fmt.Errorf("failed to read body: %v is not type of *[]byte", b)
@@ -45,14 +43,13 @@ func (c *Codec) ReadBody(b interface{}) error {
 
 func (c *Codec) Write(m *codec.Message, b interface{}) error {
 	var v []byte
-	switch b.(type) {
+	switch vb := b.(type) {
 	case *Frame:
-		v = b.(*Frame).Data
+		v = vb.Data
 	case *[]byte:
-		ve := b.(*[]byte)
-		v = *ve
+		v = *vb
 	case []byte:
-		v = b.([]byte)
+		v = vb
 	default:
 		return fmt.Errorf("failed to write: %v is not type of *[]byte or []byte", b)
 	}

codec/bytes/marshaler.go

@@ -12,25 +12,22 @@ type Message struct {
 }
 
 func (n Marshaler) Marshal(v interface{}) ([]byte, error) {
-	switch v.(type) {
+	switch ve := v.(type) {
 	case *[]byte:
-		ve := v.(*[]byte)
 		return *ve, nil
 	case []byte:
-		return v.([]byte), nil
+		return ve, nil
 	case *Message:
-		return v.(*Message).Body, nil
+		return ve.Body, nil
 	}
 	return nil, errors.New("invalid message")
 }
 
 func (n Marshaler) Unmarshal(d []byte, v interface{}) error {
-	switch v.(type) {
+	switch ve := v.(type) {
 	case *[]byte:
-		ve := v.(*[]byte)
 		*ve = d
 	case *Message:
-		ve := v.(*Message)
 		ve.Body = d
 	}
 	return errors.New("invalid message")

codec/jsonrpc/jsonrpc.go

@@ -60,7 +60,6 @@ func (j *jsonCodec) ReadHeader(m *codec.Message, mt codec.MessageType) error {
 	default:
 		return fmt.Errorf("Unrecognised message type: %v", mt)
 	}
-	return nil
 }
 
 func (j *jsonCodec) ReadBody(b interface{}) error {

codec/text/text.go

@@ -29,15 +29,12 @@ func (c *Codec) ReadBody(b interface{}) error {
 		return err
 	}
 
-	switch b.(type) {
+	switch v := b.(type) {
 	case *string:
-		v := b.(*string)
 		*v = string(buf)
 	case *[]byte:
-		v := b.(*[]byte)
 		*v = buf
 	case *Frame:
-		v := b.(*Frame)
 		v.Data = buf
 	default:
 		return fmt.Errorf("failed to read body: %v is not type of *[]byte", b)
@@ -48,20 +45,17 @@ func (c *Codec) ReadBody(b interface{}) error {
 
 func (c *Codec) Write(m *codec.Message, b interface{}) error {
 	var v []byte
-	switch b.(type) {
+	switch ve := b.(type) {
 	case *Frame:
-		v = b.(*Frame).Data
+		v = ve.Data
 	case *[]byte:
-		ve := b.(*[]byte)
 		v = *ve
 	case *string:
-		ve := b.(*string)
 		v = []byte(*ve)
 	case string:
-		ve := b.(string)
 		v = []byte(ve)
 	case []byte:
-		v = b.([]byte)
+		v = ve
 	default:
 		return fmt.Errorf("failed to write: %v is not type of *[]byte or []byte", b)
 	}

config/cmd/cmd.go

@@ -23,11 +23,11 @@ import (
 	"github.com/micro/go-micro/broker/http"
 	"github.com/micro/go-micro/broker/memory"
 	"github.com/micro/go-micro/broker/nats"
+	brokerSrv "github.com/micro/go-micro/broker/service"
 
 	// registries
 	"github.com/micro/go-micro/registry"
-	"github.com/micro/go-micro/registry/consul"
-	"github.com/micro/go-micro/registry/gossip"
+	"github.com/micro/go-micro/registry/etcd"
 	"github.com/micro/go-micro/registry/mdns"
 	rmem "github.com/micro/go-micro/registry/memory"
 	regSrv "github.com/micro/go-micro/registry/service"
@@ -44,6 +44,10 @@ import (
 	thttp "github.com/micro/go-micro/transport/http"
 	tmem "github.com/micro/go-micro/transport/memory"
 	"github.com/micro/go-micro/transport/quic"
+
+	// runtimes
+	"github.com/micro/go-micro/runtime"
+	"github.com/micro/go-micro/runtime/kubernetes"
 )
 
 type Cmd interface {
@@ -151,16 +155,27 @@ var (
 			EnvVar: "MICRO_BROKER_ADDRESS",
 			Usage:  "Comma-separated list of broker addresses",
 		},
+		cli.StringFlag{
+			Name:   "profile",
+			Usage:  "Debug profiler for cpu and memory stats",
+			EnvVar: "MICRO_DEBUG_PROFILE",
+		},
 		cli.StringFlag{
 			Name:   "registry",
 			EnvVar: "MICRO_REGISTRY",
-			Usage:  "Registry for discovery. consul, mdns",
+			Usage:  "Registry for discovery. etcd, mdns",
 		},
 		cli.StringFlag{
 			Name:   "registry_address",
 			EnvVar: "MICRO_REGISTRY_ADDRESS",
 			Usage:  "Comma-separated list of registry addresses",
 		},
+		cli.StringFlag{
+			Name:   "runtime",
+			Usage:  "Runtime for building and running services e.g local, kubernetes",
+			EnvVar: "MICRO_RUNTIME",
+			Value:  "local",
+		},
 		cli.StringFlag{
 			Name:   "selector",
 			EnvVar: "MICRO_SELECTOR",
@@ -179,9 +194,11 @@ var (
 	}
 
 	DefaultBrokers = map[string]func(...broker.Option) broker.Broker{
-		"http":   http.NewBroker,
-		"memory": memory.NewBroker,
-		"nats":   nats.NewBroker,
+		"go.micro.broker": brokerSrv.NewBroker,
+		"service":         brokerSrv.NewBroker,
+		"http":            http.NewBroker,
+		"memory":          memory.NewBroker,
+		"nats":            nats.NewBroker,
 	}
 
 	DefaultClients = map[string]func(...client.Option) client.Client{
@@ -193,8 +210,7 @@ var (
 	DefaultRegistries = map[string]func(...registry.Option) registry.Registry{
 		"go.micro.registry": regSrv.NewRegistry,
 		"service":           regSrv.NewRegistry,
-		"consul":            consul.NewRegistry,
-		"gossip":            gossip.NewRegistry,
+		"etcd":              etcd.NewRegistry,
 		"mdns":              mdns.NewRegistry,
 		"memory":            rmem.NewRegistry,
 	}
@@ -220,6 +236,11 @@ var (
 		"quic":   quic.NewTransport,
 	}
 
+	DefaultRuntimes = map[string]func(...runtime.Option) runtime.Runtime{
+		"local":      runtime.NewRuntime,
+		"kubernetes": kubernetes.NewRuntime,
+	}
+
 	// used for default selection as the fall back
 	defaultClient    = "rpc"
 	defaultServer    = "rpc"
@@ -227,6 +248,7 @@ var (
 	defaultRegistry  = "mdns"
 	defaultSelector  = "registry"
 	defaultTransport = "http"
+	defaultRuntime   = "local"
 )
 
 func init() {
@@ -246,6 +268,7 @@ func newCmd(opts ...Option) Cmd {
 		Server:    &server.DefaultServer,
 		Selector:  &selector.DefaultSelector,
 		Transport: &transport.DefaultTransport,
+		Runtime:   &runtime.DefaultRuntime,
 
 		Brokers:    DefaultBrokers,
 		Clients:    DefaultClients,
@@ -253,6 +276,7 @@ func newCmd(opts ...Option) Cmd {
 		Selectors:  DefaultSelectors,
 		Servers:    DefaultServers,
 		Transports: DefaultTransports,
+		Runtimes:   DefaultRuntimes,
 	}
 
 	for _, o := range opts {
@@ -293,6 +317,16 @@ func (c *cmd) Before(ctx *cli.Context) error {
 	var serverOpts []server.Option
 	var clientOpts []client.Option
 
+	// Set the runtime
+	if name := ctx.String("runtime"); len(name) > 0 {
+		r, ok := c.opts.Runtimes[name]
+		if !ok {
+			return fmt.Errorf("Unsupported runtime: %s", name)
+		}
+
+		*c.opts.Runtime = r()
+	}
+
 	// Set the client
 	if name := ctx.String("client"); len(name) > 0 {
 		// only change if we have the client and type differs

config/cmd/options.go

@@ -7,6 +7,7 @@ import (
 	"github.com/micro/go-micro/client"
 	"github.com/micro/go-micro/client/selector"
 	"github.com/micro/go-micro/registry"
+	"github.com/micro/go-micro/runtime"
 	"github.com/micro/go-micro/server"
 	"github.com/micro/go-micro/transport"
 )
@@ -24,6 +25,7 @@ type Options struct {
 	Transport *transport.Transport
 	Client    *client.Client
 	Server    *server.Server
+	Runtime   *runtime.Runtime
 
 	Brokers    map[string]func(...broker.Option) broker.Broker
 	Clients    map[string]func(...client.Option) client.Client
@@ -31,6 +33,7 @@ type Options struct {
 	Selectors  map[string]func(...selector.Option) selector.Selector
 	Servers    map[string]func(...server.Option) server.Server
 	Transports map[string]func(...transport.Option) transport.Transport
+	Runtimes   map[string]func(...runtime.Option) runtime.Runtime
 
 	// Other options for implementations of the interface
 	// can be stored in a context
@@ -135,3 +138,10 @@ func NewTransport(name string, t func(...transport.Option) transport.Transport)
 		o.Transports[name] = t
 	}
 }
+
+// New runtime func
+func NewRuntime(name string, r func(...runtime.Option) runtime.Runtime) Option {
+	return func(o *Options) {
+		o.Runtimes[name] = r
+	}
+}

config/loader/memory/memory.go

@@ -2,6 +2,7 @@ package memory
 
 import (
 	"bytes"
+	"container/list"
 	"errors"
 	"fmt"
 	"strings"
@@ -28,8 +29,7 @@ type memory struct {
 	// all the sources
 	sources []source.Source
 
-	idx      int
-	watchers map[int]*watcher
+	watchers *list.List
 }
 
 type watcher struct {
@@ -153,11 +153,11 @@ func (m *memory) reload() error {
 }
 
 func (m *memory) update() {
-	var watchers []*watcher
+	watchers := make([]*watcher, 0, m.watchers.Len())
 
 	m.RLock()
-	for _, w := range m.watchers {
-		watchers = append(watchers, w)
+	for e := m.watchers.Front(); e != nil; e = e.Next() {
+		watchers = append(watchers, e.Value.(*watcher))
 	}
 	m.RUnlock()
 
@@ -193,6 +193,7 @@ func (m *memory) Snapshot() (*loader.Snapshot, error) {
 
 // Sync loads all the sources, calls the parser and updates the config
 func (m *memory) Sync() error {
+	//nolint:prealloc
 	var sets []*source.ChangeSet
 
 	m.Lock()
@@ -335,16 +336,14 @@ func (m *memory) Watch(path ...string) (loader.Watcher, error) {
 		updates: make(chan reader.Value, 1),
 	}
 
-	id := m.idx
-	m.watchers[id] = w
-	m.idx++
+	e := m.watchers.PushBack(w)
 
 	m.Unlock()
 
 	go func() {
 		<-w.exit
 		m.Lock()
-		delete(m.watchers, id)
+		m.watchers.Remove(e)
 		m.Unlock()
 	}()
 
@@ -403,7 +402,7 @@ func NewLoader(opts ...loader.Option) loader.Loader {
 	m := &memory{
 		exit:     make(chan bool),
 		opts:     options,
-		watchers: make(map[int]*watcher),
+		watchers: list.New(),
 		sources:  options.Source,
 	}
 

config/options/options.go

@@ -2,6 +2,7 @@
 package options
 
 import (
+	"log"
 	"sync"
 )
 
@@ -68,6 +69,8 @@ func WithString(s string) Option {
 // NewOptions returns a new initialiser
 func NewOptions(opts ...Option) Options {
 	o := new(defaultOptions)
-	o.Init(opts...)
+	if err := o.Init(opts...); err != nil {
+		log.Fatal(err)
+	}
 	return o
 }

config/reader/json/values_test.go

@@ -62,7 +62,7 @@ func TestStructArray(t *testing.T) {
 		{
 			[]byte(`[{"foo": "bar"}]`),
 			emptyTSlice,
-			[]T{T{Foo: "bar"}},
+			[]T{{Foo: "bar"}},
 		},
 	}
 

config/source/consul/README.md

@@ -1,49 +0,0 @@
-# Consul Source
-
-The consul source reads config from consul key/values
-
-## Consul Format
-
-The consul source expects keys under the default prefix `/micro/config`
-
-Values are expected to be json
-
-```
-// set database
-consul kv put micro/config/database '{"address": "10.0.0.1", "port": 3306}'
-// set cache
-consul kv put micro/config/cache '{"address": "10.0.0.2", "port": 6379}'
-```
-
-Keys are split on `/` so access becomes
-
-```
-conf.Get("micro", "config", "database")
-```
-
-## New Source
-
-Specify source with data
-
-```go
-consulSource := consul.NewSource(
-	// optionally specify consul address; default to localhost:8500
-	consul.WithAddress("10.0.0.10:8500"),
-	// optionally specify prefix; defaults to /micro/config
-	consul.WithPrefix("/my/prefix"),
-  // optionally strip the provided prefix from the keys, defaults to false
-  consul.StripPrefix(true),
-)
-```
-
-## Load Source
-
-Load the source into config
-
-```go
-// Create new config
-conf := config.NewConfig()
-
-// Load consul source
-conf.Load(consulSource)
-```

config/source/consul/consul.go

@@ -1,126 +0,0 @@
-package consul
-
-import (
-	"fmt"
-	"net"
-	"time"
-
-	"github.com/hashicorp/consul/api"
-	"github.com/micro/go-micro/config/source"
-)
-
-// Currently a single consul reader
-type consul struct {
-	prefix      string
-	stripPrefix string
-	addr        string
-	opts        source.Options
-	client      *api.Client
-}
-
-var (
-	// DefaultPrefix is the prefix that consul keys will be assumed to have if you
-	// haven't specified one
-	DefaultPrefix = "/micro/config/"
-)
-
-func (c *consul) Read() (*source.ChangeSet, error) {
-	kv, _, err := c.client.KV().List(c.prefix, nil)
-	if err != nil {
-		return nil, err
-	}
-
-	if kv == nil || len(kv) == 0 {
-		return nil, fmt.Errorf("source not found: %s", c.prefix)
-	}
-
-	data, err := makeMap(c.opts.Encoder, kv, c.stripPrefix)
-	if err != nil {
-		return nil, fmt.Errorf("error reading data: %v", err)
-	}
-
-	b, err := c.opts.Encoder.Encode(data)
-	if err != nil {
-		return nil, fmt.Errorf("error reading source: %v", err)
-	}
-
-	cs := &source.ChangeSet{
-		Timestamp: time.Now(),
-		Format:    c.opts.Encoder.String(),
-		Source:    c.String(),
-		Data:      b,
-	}
-	cs.Checksum = cs.Sum()
-
-	return cs, nil
-}
-
-func (c *consul) String() string {
-	return "consul"
-}
-
-func (c *consul) Watch() (source.Watcher, error) {
-	w, err := newWatcher(c.prefix, c.addr, c.String(), c.stripPrefix, c.opts.Encoder)
-	if err != nil {
-		return nil, err
-	}
-	return w, nil
-}
-
-// NewSource creates a new consul source
-func NewSource(opts ...source.Option) source.Source {
-	options := source.NewOptions(opts...)
-
-	// use default config
-	config := api.DefaultConfig()
-
-	// use the consul config passed in the options if any
-	if co, ok := options.Context.Value(configKey{}).(*api.Config); ok {
-		config = co
-	}
-
-	// check if there are any addrs
-	a, ok := options.Context.Value(addressKey{}).(string)
-	if ok {
-		addr, port, err := net.SplitHostPort(a)
-		if ae, ok := err.(*net.AddrError); ok && ae.Err == "missing port in address" {
-			port = "8500"
-			addr = a
-			config.Address = fmt.Sprintf("%s:%s", addr, port)
-		} else if err == nil {
-			config.Address = fmt.Sprintf("%s:%s", addr, port)
-		}
-	}
-
-	dc, ok := options.Context.Value(dcKey{}).(string)
-	if ok {
-		config.Datacenter = dc
-	}
-
-	token, ok := options.Context.Value(tokenKey{}).(string)
-	if ok {
-		config.Token = token
-	}
-
-	// create the client
-	client, _ := api.NewClient(config)
-
-	prefix := DefaultPrefix
-	sp := ""
-	f, ok := options.Context.Value(prefixKey{}).(string)
-	if ok {
-		prefix = f
-	}
-
-	if b, ok := options.Context.Value(stripPrefixKey{}).(bool); ok && b {
-		sp = prefix
-	}
-
-	return &consul{
-		prefix:      prefix,
-		stripPrefix: sp,
-		addr:        config.Address,
-		opts:        options,
-		client:      client,
-	}
-}

config/source/consul/util.go

@@ -1,89 +0,0 @@
-package consul
-
-import (
-	"fmt"
-	"strings"
-
-	"github.com/hashicorp/consul/api"
-	"github.com/micro/go-micro/config/encoder"
-)
-
-type configValue interface {
-	Value() interface{}
-	Decode(encoder.Encoder, []byte) error
-}
-type configArrayValue struct {
-	v []interface{}
-}
-
-func (a *configArrayValue) Value() interface{} { return a.v }
-func (a *configArrayValue) Decode(e encoder.Encoder, b []byte) error {
-	return e.Decode(b, &a.v)
-}
-
-type configMapValue struct {
-	v map[string]interface{}
-}
-
-func (m *configMapValue) Value() interface{} { return m.v }
-func (m *configMapValue) Decode(e encoder.Encoder, b []byte) error {
-	return e.Decode(b, &m.v)
-}
-
-func makeMap(e encoder.Encoder, kv api.KVPairs, stripPrefix string) (map[string]interface{}, error) {
-
-	data := make(map[string]interface{})
-
-	// consul guarantees lexicographic order, so no need to sort
-	for _, v := range kv {
-		pathString := strings.TrimPrefix(strings.TrimPrefix(v.Key, strings.TrimPrefix(stripPrefix, "/")), "/")
-		if pathString == "" {
-			continue
-		}
-		var val configValue
-		var err error
-
-		// ensure a valid value is stored at this location
-		if len(v.Value) > 0 {
-			// try to decode into map value or array value
-			arrayV := &configArrayValue{v: []interface{}{}}
-			mapV := &configMapValue{v: map[string]interface{}{}}
-			switch {
-			case arrayV.Decode(e, v.Value) == nil:
-				val = arrayV
-			case mapV.Decode(e, v.Value) == nil:
-				val = mapV
-			default:
-				return nil, fmt.Errorf("faild decode value. path: %s, error: %s", pathString, err)
-			}
-		}
-
-		// set target at the root
-		target := data
-		path := strings.Split(pathString, "/")
-		// find (or create) the leaf node we want to put this value at
-		for _, dir := range path[:len(path)-1] {
-			if _, ok := target[dir]; !ok {
-				target[dir] = make(map[string]interface{})
-			}
-			target = target[dir].(map[string]interface{})
-		}
-
-		leafDir := path[len(path)-1]
-
-		// copy over the keys from the value
-		switch val.(type) {
-		case *configArrayValue:
-			target[leafDir] = val.Value()
-		case *configMapValue:
-			target[leafDir] = make(map[string]interface{})
-			target = target[leafDir].(map[string]interface{})
-			mapv := val.Value().(map[string]interface{})
-			for k := range mapv {
-				target[k] = mapv[k]
-			}
-		}
-	}
-
-	return data, nil
-}

config/source/consul/watcher.go

@@ -1,95 +0,0 @@
-package consul
-
-import (
-	"time"
-
-	"github.com/hashicorp/consul/api"
-	"github.com/hashicorp/consul/api/watch"
-	"github.com/micro/go-micro/config/encoder"
-	"github.com/micro/go-micro/config/source"
-)
-
-type watcher struct {
-	e           encoder.Encoder
-	name        string
-	stripPrefix string
-
-	wp   *watch.Plan
-	ch   chan *source.ChangeSet
-	exit chan bool
-}
-
-func newWatcher(key, addr, name, stripPrefix string, e encoder.Encoder) (source.Watcher, error) {
-	w := &watcher{
-		e:           e,
-		name:        name,
-		stripPrefix: stripPrefix,
-		ch:          make(chan *source.ChangeSet),
-		exit:        make(chan bool),
-	}
-
-	wp, err := watch.Parse(map[string]interface{}{"type": "keyprefix", "prefix": key})
-	if err != nil {
-		return nil, err
-	}
-
-	wp.Handler = w.handle
-
-	// wp.Run is a blocking call and will prevent newWatcher from returning
-	go wp.Run(addr)
-
-	w.wp = wp
-
-	return w, nil
-}
-
-func (w *watcher) handle(idx uint64, data interface{}) {
-	if data == nil {
-		return
-	}
-
-	kvs, ok := data.(api.KVPairs)
-	if !ok {
-		return
-	}
-
-	d, err := makeMap(w.e, kvs, w.stripPrefix)
-	if err != nil {
-		return
-	}
-
-	b, err := w.e.Encode(d)
-	if err != nil {
-		return
-	}
-
-	cs := &source.ChangeSet{
-		Timestamp: time.Now(),
-		Format:    w.e.String(),
-		Source:    w.name,
-		Data:      b,
-	}
-	cs.Checksum = cs.Sum()
-
-	w.ch <- cs
-}
-
-func (w *watcher) Next() (*source.ChangeSet, error) {
-	select {
-	case cs := <-w.ch:
-		return cs, nil
-	case <-w.exit:
-		return nil, source.ErrWatcherStopped
-	}
-}
-
-func (w *watcher) Stop() error {
-	select {
-	case <-w.exit:
-		return nil
-	default:
-		w.wp.Stop()
-		close(w.exit)
-	}
-	return nil
-}

config/source/env/options.go

@@ -35,6 +35,7 @@ func WithPrefix(p ...string) source.Option {
 }
 
 func appendUnderscore(prefixes []string) []string {
+	//nolint:prealloc
 	var result []string
 	for _, p := range prefixes {
 		if !strings.HasSuffix(p, "_") {

config/source/etcd/README.md

@@ -0,0 +1,51 @@
+# Etcd Source
+
+The etcd source reads config from etcd key/values
+
+This source supports etcd version 3 and beyond.
+
+## Etcd Format
+
+The etcd source expects keys under the default prefix `/micro/config` (prefix can be changed)
+
+Values are expected to be JSON
+
+```
+// set database
+etcdctl put /micro/config/database '{"address": "10.0.0.1", "port": 3306}'
+// set cache
+etcdctl put /micro/config/cache '{"address": "10.0.0.2", "port": 6379}'
+```
+
+Keys are split on `/` so access becomes
+
+```
+conf.Get("micro", "config", "database")
+```
+
+## New Source
+
+Specify source with data
+
+```go
+etcdSource := etcd.NewSource(
+	// optionally specify etcd address; default to localhost:8500
+	etcd.WithAddress("10.0.0.10:8500"),
+	// optionally specify prefix; defaults to /micro/config
+	etcd.WithPrefix("/my/prefix"),
+	// optionally strip the provided prefix from the keys, defaults to false
+	etcd.StripPrefix(true),
+)
+```
+
+## Load Source
+
+Load the source into config
+
+```go
+// Create new config
+conf := config.NewConfig()
+
+// Load file source
+conf.Load(etcdSource)
+```

config/source/etcd/etcd.go

@@ -0,0 +1,141 @@
+package etcd
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"time"
+
+	cetcd "github.com/coreos/etcd/clientv3"
+	"github.com/coreos/etcd/mvcc/mvccpb"
+	"github.com/micro/go-micro/config/source"
+)
+
+// Currently a single etcd reader
+type etcd struct {
+	prefix      string
+	stripPrefix string
+	opts        source.Options
+	client      *cetcd.Client
+	cerr        error
+}
+
+var (
+	DefaultPrefix = "/micro/config/"
+)
+
+func (c *etcd) Read() (*source.ChangeSet, error) {
+	if c.cerr != nil {
+		return nil, c.cerr
+	}
+
+	rsp, err := c.client.Get(context.Background(), c.prefix, cetcd.WithPrefix())
+	if err != nil {
+		return nil, err
+	}
+
+	if rsp == nil || len(rsp.Kvs) == 0 {
+		return nil, fmt.Errorf("source not found: %s", c.prefix)
+	}
+
+	kvs := make([]*mvccpb.KeyValue, 0, len(rsp.Kvs))
+	for _, v := range rsp.Kvs {
+		kvs = append(kvs, (*mvccpb.KeyValue)(v))
+	}
+
+	data := makeMap(c.opts.Encoder, kvs, c.stripPrefix)
+
+	b, err := c.opts.Encoder.Encode(data)
+	if err != nil {
+		return nil, fmt.Errorf("error reading source: %v", err)
+	}
+
+	cs := &source.ChangeSet{
+		Timestamp: time.Now(),
+		Source:    c.String(),
+		Data:      b,
+		Format:    c.opts.Encoder.String(),
+	}
+	cs.Checksum = cs.Sum()
+
+	return cs, nil
+}
+
+func (c *etcd) String() string {
+	return "etcd"
+}
+
+func (c *etcd) Watch() (source.Watcher, error) {
+	if c.cerr != nil {
+		return nil, c.cerr
+	}
+	cs, err := c.Read()
+	if err != nil {
+		return nil, err
+	}
+	return newWatcher(c.prefix, c.stripPrefix, c.client.Watcher, cs, c.opts)
+}
+
+func NewSource(opts ...source.Option) source.Source {
+	options := source.NewOptions(opts...)
+
+	var endpoints []string
+
+	// check if there are any addrs
+	addrs, ok := options.Context.Value(addressKey{}).([]string)
+	if ok {
+		for _, a := range addrs {
+			addr, port, err := net.SplitHostPort(a)
+			if ae, ok := err.(*net.AddrError); ok && ae.Err == "missing port in address" {
+				port = "2379"
+				addr = a
+				endpoints = append(endpoints, fmt.Sprintf("%s:%s", addr, port))
+			} else if err == nil {
+				endpoints = append(endpoints, fmt.Sprintf("%s:%s", addr, port))
+			}
+		}
+	}
+
+	if len(endpoints) == 0 {
+		endpoints = []string{"localhost:2379"}
+	}
+
+	// check dial timeout option
+	dialTimeout, ok := options.Context.Value(dialTimeoutKey{}).(time.Duration)
+	if !ok {
+		dialTimeout = 3 * time.Second // default dial timeout
+	}
+
+	config := cetcd.Config{
+		Endpoints:   endpoints,
+		DialTimeout: dialTimeout,
+	}
+
+	u, ok := options.Context.Value(authKey{}).(*authCreds)
+	if ok {
+		config.Username = u.Username
+		config.Password = u.Password
+	}
+
+	// use default config
+	client, err := cetcd.New(config)
+
+	prefix := DefaultPrefix
+	sp := ""
+	f, ok := options.Context.Value(prefixKey{}).(string)
+	if ok {
+		prefix = f
+	}
+
+	if b, ok := options.Context.Value(stripPrefixKey{}).(bool); ok && b {
+		sp = prefix
+	}
+
+	return &etcd{
+		prefix:      prefix,
+		stripPrefix: sp,
+		opts:        options,
+		client:      client,
+		cerr:        err,
+	}
+}

config/source/etcd/options.go

@@ -1,21 +1,25 @@
-package consul
+package etcd
 
 import (
 	"context"
+	"time"
 
-	"github.com/hashicorp/consul/api"
 	"github.com/micro/go-micro/config/source"
 )
 
 type addressKey struct{}
 type prefixKey struct{}
 type stripPrefixKey struct{}
-type dcKey struct{}
-type tokenKey struct{}
-type configKey struct{}
+type authKey struct{}
+type dialTimeoutKey struct{}
 
-// WithAddress sets the consul address
-func WithAddress(a string) source.Option {
+type authCreds struct {
+	Username string
+	Password string
+}
+
+// WithAddress sets the etcd address
+func WithAddress(a ...string) source.Option {
 	return func(o *source.Options) {
 		if o.Context == nil {
 			o.Context = context.Background()
@@ -45,31 +49,22 @@ func StripPrefix(strip bool) source.Option {
 	}
 }
 
-func WithDatacenter(p string) source.Option {
+// Auth allows you to specify username/password
+func Auth(username, password string) source.Option {
 	return func(o *source.Options) {
 		if o.Context == nil {
 			o.Context = context.Background()
 		}
-		o.Context = context.WithValue(o.Context, dcKey{}, p)
+		o.Context = context.WithValue(o.Context, authKey{}, &authCreds{Username: username, Password: password})
 	}
 }
 
-// WithToken sets the key token to use
-func WithToken(p string) source.Option {
+// WithDialTimeout set the time out for dialing to etcd
+func WithDialTimeout(timeout time.Duration) source.Option {
 	return func(o *source.Options) {
 		if o.Context == nil {
 			o.Context = context.Background()
 		}
-		o.Context = context.WithValue(o.Context, tokenKey{}, p)
-	}
-}
-
-// WithConfig set consul-specific options
-func WithConfig(c *api.Config) source.Option {
-	return func(o *source.Options) {
-		if o.Context == nil {
-			o.Context = context.Background()
-		}
-		o.Context = context.WithValue(o.Context, configKey{}, c)
+		o.Context = context.WithValue(o.Context, dialTimeoutKey{}, timeout)
 	}
 }

config/source/etcd/util.go

@@ -0,0 +1,89 @@
+package etcd
+
+import (
+	"strings"
+
+	"github.com/coreos/etcd/clientv3"
+	"github.com/coreos/etcd/mvcc/mvccpb"
+	"github.com/micro/go-micro/config/encoder"
+)
+
+func makeEvMap(e encoder.Encoder, data map[string]interface{}, kv []*clientv3.Event, stripPrefix string) map[string]interface{} {
+	if data == nil {
+		data = make(map[string]interface{})
+	}
+
+	for _, v := range kv {
+		switch mvccpb.Event_EventType(v.Type) {
+		case mvccpb.DELETE:
+			data = update(e, data, (*mvccpb.KeyValue)(v.Kv), "delete", stripPrefix)
+		default:
+			data = update(e, data, (*mvccpb.KeyValue)(v.Kv), "insert", stripPrefix)
+		}
+	}
+
+	return data
+}
+
+func makeMap(e encoder.Encoder, kv []*mvccpb.KeyValue, stripPrefix string) map[string]interface{} {
+	data := make(map[string]interface{})
+
+	for _, v := range kv {
+		data = update(e, data, v, "put", stripPrefix)
+	}
+
+	return data
+}
+
+func update(e encoder.Encoder, data map[string]interface{}, v *mvccpb.KeyValue, action, stripPrefix string) map[string]interface{} {
+	// remove prefix if non empty, and ensure leading / is removed as well
+	vkey := strings.TrimPrefix(strings.TrimPrefix(string(v.Key), stripPrefix), "/")
+	// split on prefix
+	haveSplit := strings.Contains(vkey, "/")
+	keys := strings.Split(vkey, "/")
+
+	var vals interface{}
+	e.Decode(v.Value, &vals)
+
+	if !haveSplit && len(keys) == 1 {
+		switch action {
+		case "delete":
+			data = make(map[string]interface{})
+		default:
+			v, ok := vals.(map[string]interface{})
+			if ok {
+				data = v
+			}
+		}
+		return data
+	}
+
+	// set data for first iteration
+	kvals := data
+	// iterate the keys and make maps
+	for i, k := range keys {
+		kval, ok := kvals[k].(map[string]interface{})
+		if !ok {
+			// create next map
+			kval = make(map[string]interface{})
+			// set it
+			kvals[k] = kval
+		}
+
+		// last key: write vals
+		if l := len(keys) - 1; i == l {
+			switch action {
+			case "delete":
+				delete(kvals, k)
+			default:
+				kvals[k] = vals
+			}
+			break
+		}
+
+		// set kvals for next iterator
+		kvals = kval
+	}
+
+	return data
+}

config/source/etcd/watcher.go

@@ -0,0 +1,113 @@
+package etcd
+
+import (
+	"context"
+	"errors"
+	"sync"
+	"time"
+
+	cetcd "github.com/coreos/etcd/clientv3"
+	"github.com/micro/go-micro/config/source"
+)
+
+type watcher struct {
+	opts        source.Options
+	name        string
+	stripPrefix string
+
+	sync.RWMutex
+	cs *source.ChangeSet
+
+	ch   chan *source.ChangeSet
+	exit chan bool
+}
+
+func newWatcher(key, strip string, wc cetcd.Watcher, cs *source.ChangeSet, opts source.Options) (source.Watcher, error) {
+	w := &watcher{
+		opts:        opts,
+		name:        "etcd",
+		stripPrefix: strip,
+		cs:          cs,
+		ch:          make(chan *source.ChangeSet),
+		exit:        make(chan bool),
+	}
+
+	ch := wc.Watch(context.Background(), key, cetcd.WithPrefix())
+
+	go w.run(wc, ch)
+
+	return w, nil
+}
+
+func (w *watcher) handle(evs []*cetcd.Event) {
+	w.RLock()
+	data := w.cs.Data
+	w.RUnlock()
+
+	var vals map[string]interface{}
+
+	// unpackage existing changeset
+	if err := w.opts.Encoder.Decode(data, &vals); err != nil {
+		return
+	}
+
+	// update base changeset
+	d := makeEvMap(w.opts.Encoder, vals, evs, w.stripPrefix)
+
+	// pack the changeset
+	b, err := w.opts.Encoder.Encode(d)
+	if err != nil {
+		return
+	}
+
+	// create new changeset
+	cs := &source.ChangeSet{
+		Timestamp: time.Now(),
+		Source:    w.name,
+		Data:      b,
+		Format:    w.opts.Encoder.String(),
+	}
+	cs.Checksum = cs.Sum()
+
+	// set base change set
+	w.Lock()
+	w.cs = cs
+	w.Unlock()
+
+	// send update
+	w.ch <- cs
+}
+
+func (w *watcher) run(wc cetcd.Watcher, ch cetcd.WatchChan) {
+	for {
+		select {
+		case rsp, ok := <-ch:
+			if !ok {
+				return
+			}
+			w.handle(rsp.Events)
+		case <-w.exit:
+			wc.Close()
+			return
+		}
+	}
+}
+
+func (w *watcher) Next() (*source.ChangeSet, error) {
+	select {
+	case cs := <-w.ch:
+		return cs, nil
+	case <-w.exit:
+		return nil, errors.New("watcher stopped")
+	}
+}
+
+func (w *watcher) Stop() error {
+	select {
+	case <-w.exit:
+		return nil
+	default:
+		close(w.exit)
+	}
+	return nil
+}

data/data.go

@@ -1,2 +0,0 @@
-// Package data is an interface for data access
-package data

data/store/consul/consul.go

@@ -1,96 +0,0 @@
-// Package consul is a consul implementation of kv
-package consul
-
-import (
-	"fmt"
-	"net"
-
-	"github.com/hashicorp/consul/api"
-	"github.com/micro/go-micro/config/options"
-	"github.com/micro/go-micro/data/store"
-)
-
-type ckv struct {
-	options.Options
-	client *api.Client
-}
-
-func (c *ckv) Read(key string) (*store.Record, error) {
-	keyval, _, err := c.client.KV().Get(key, nil)
-	if err != nil {
-		return nil, err
-	}
-
-	if keyval == nil {
-		return nil, store.ErrNotFound
-	}
-
-	return &store.Record{
-		Key:   keyval.Key,
-		Value: keyval.Value,
-	}, nil
-}
-
-func (c *ckv) Delete(key string) error {
-	_, err := c.client.KV().Delete(key, nil)
-	return err
-}
-
-func (c *ckv) Write(record *store.Record) error {
-	_, err := c.client.KV().Put(&api.KVPair{
-		Key:   record.Key,
-		Value: record.Value,
-	}, nil)
-	return err
-}
-
-func (c *ckv) Dump() ([]*store.Record, error) {
-	keyval, _, err := c.client.KV().List("/", nil)
-	if err != nil {
-		return nil, err
-	}
-	if keyval == nil {
-		return nil, store.ErrNotFound
-	}
-	var vals []*store.Record
-	for _, keyv := range keyval {
-		vals = append(vals, &store.Record{
-			Key:   keyv.Key,
-			Value: keyv.Value,
-		})
-	}
-	return vals, nil
-}
-
-func (c *ckv) String() string {
-	return "consul"
-}
-
-func NewStore(opts ...options.Option) store.Store {
-	options := options.NewOptions(opts...)
-	config := api.DefaultConfig()
-
-	var nodes []string
-
-	if n, ok := options.Values().Get("store.nodes"); ok {
-		nodes = n.([]string)
-	}
-
-	// set host
-	if len(nodes) > 0 {
-		addr, port, err := net.SplitHostPort(nodes[0])
-		if ae, ok := err.(*net.AddrError); ok && ae.Err == "missing port in address" {
-			port = "8500"
-			config.Address = fmt.Sprintf("%s:%s", nodes[0], port)
-		} else if err == nil {
-			config.Address = fmt.Sprintf("%s:%s", addr, port)
-		}
-	}
-
-	client, _ := api.NewClient(config)
-
-	return &ckv{
-		Options: options,
-		client:  client,
-	}
-}

debug/buffer/buffer.go

@@ -0,0 +1,136 @@
+// Package buffer provides a simple ring buffer for storing local data
+package buffer
+
+import (
+	"sync"
+	"time"
+
+	"github.com/google/uuid"
+)
+
+type stream struct {
+	id      string
+	entries chan *Entry
+	stop    chan bool
+}
+
+// Buffer is ring buffer
+type Buffer struct {
+	size int
+	sync.RWMutex
+	vals    []*Entry
+	streams map[string]stream
+}
+
+// Entry is ring buffer data entry
+type Entry struct {
+	Value     interface{}
+	Timestamp time.Time
+}
+
+// New returns a new buffer of the given size
+func New(i int) *Buffer {
+	return &Buffer{
+		size:    i,
+		streams: make(map[string]stream),
+	}
+}
+
+// Put adds a new value to ring buffer
+func (b *Buffer) Put(v interface{}) {
+	b.Lock()
+	defer b.Unlock()
+
+	// append to values
+	entry := &Entry{
+		Value:     v,
+		Timestamp: time.Now(),
+	}
+	b.vals = append(b.vals, entry)
+
+	// trim if bigger than size required
+	if len(b.vals) > b.size {
+		b.vals = b.vals[1:]
+	}
+
+	// TODO: this is fucking ugly
+	for _, stream := range b.streams {
+		select {
+		case <-stream.stop:
+			delete(b.streams, stream.id)
+			close(stream.entries)
+		case stream.entries <- entry:
+		}
+	}
+}
+
+// Get returns the last n entries
+func (b *Buffer) Get(n int) []*Entry {
+	b.RLock()
+	defer b.RUnlock()
+
+	// reset any invalid values
+	if n > b.size || n < 0 {
+		n = b.size
+	}
+
+	// create a delta
+	delta := b.size - n
+
+	// if all the values are less than delta
+	if len(b.vals) < delta {
+		return b.vals
+	}
+
+	// return the delta set
+	return b.vals[delta:]
+}
+
+// Return the entries since a specific time
+func (b *Buffer) Since(t time.Time) []*Entry {
+	b.RLock()
+	defer b.RUnlock()
+
+	// return all the values
+	if t.IsZero() {
+		return b.vals
+	}
+
+	// if its in the future return nothing
+	if time.Since(t).Seconds() < 0.0 {
+		return nil
+	}
+
+	for i, v := range b.vals {
+		// find the starting point
+		d := v.Timestamp.Sub(t)
+
+		// return the values
+		if d.Seconds() > 0.0 {
+			return b.vals[i:]
+		}
+	}
+
+	return nil
+}
+
+// Stream logs from the buffer
+func (b *Buffer) Stream(stop chan bool) <-chan *Entry {
+	b.Lock()
+	defer b.Unlock()
+
+	entries := make(chan *Entry, 128)
+	id := uuid.New().String()
+	b.streams[id] = stream{
+		id:      id,
+		entries: entries,
+		stop:    stop,
+	}
+
+	return entries
+}
+
+// Size returns the size of the ring buffer
+func (b *Buffer) Size() int {
+	return b.size
+}

debug/buffer/buffer_test.go

@@ -0,0 +1,79 @@
+package buffer
+
+import (
+	"testing"
+	"time"
+)
+
+func TestBuffer(t *testing.T) {
+	b := New(10)
+
+	// test one value
+	b.Put("foo")
+	v := b.Get(1)
+
+	if val := v[0].Value.(string); val != "foo" {
+		t.Fatalf("expected foo got %v", val)
+	}
+
+	b = New(10)
+
+	// test 10 values
+	for i := 0; i < 10; i++ {
+		b.Put(i)
+	}
+
+	d := time.Now()
+	v = b.Get(10)
+
+	for i := 0; i < 10; i++ {
+		val := v[i].Value.(int)
+
+		if val != i {
+			t.Fatalf("expected %d got %d", i, val)
+		}
+	}
+
+	// test more values
+
+	for i := 0; i < 10; i++ {
+		v := i * 2
+		b.Put(v)
+	}
+
+	v = b.Get(10)
+
+	for i := 0; i < 10; i++ {
+		val := v[i].Value.(int)
+		expect := i * 2
+		if val != expect {
+			t.Fatalf("expected %d got %d", expect, val)
+		}
+	}
+
+	// sleep 100 ms
+	time.Sleep(time.Millisecond * 100)
+
+	// assume we'll get everything
+	v = b.Since(d)
+
+	if len(v) != 10 {
+		t.Fatalf("expected 10 entries but got %d", len(v))
+	}
+
+	// write 1 more entry
+	d = time.Now()
+	b.Put(100)
+
+	// sleep 100 ms
+	time.Sleep(time.Millisecond * 100)
+
+	v = b.Since(d)
+	if len(v) != 1 {
+		t.Fatalf("expected 1 entries but got %d", len(v))
+	}
+
+	if v[0].Value.(int) != 100 {
+		t.Fatalf("expected value 100 got %v", v[0])
+	}
+}

debug/debug.go

@@ -0,0 +1,7 @@
+// Package debug provides micro debug packages
+package debug
+
+var (
+	// DefaultName is the name of debug service
+	DefaultName = "go.micro.debug"
+)

debug/handler/debug.go

@@ -1,41 +0,0 @@
-package handler
-
-import (
-	"context"
-	"runtime"
-	"time"
-
-	proto "github.com/micro/go-micro/debug/proto"
-)
-
-type Debug struct {
-	proto.DebugHandler
-	started int64
-}
-
-var (
-	DefaultHandler = newDebug()
-)
-
-func newDebug() *Debug {
-	return &Debug{
-		started: time.Now().Unix(),
-	}
-}
-
-func (d *Debug) Health(ctx context.Context, req *proto.HealthRequest, rsp *proto.HealthResponse) error {
-	rsp.Status = "ok"
-	return nil
-}
-
-func (d *Debug) Stats(ctx context.Context, req *proto.StatsRequest, rsp *proto.StatsResponse) error {
-	var mstat runtime.MemStats
-	runtime.ReadMemStats(&mstat)
-
-	rsp.Started = uint64(d.started)
-	rsp.Uptime = uint64(time.Now().Unix() - d.started)
-	rsp.Memory = mstat.Alloc
-	rsp.Gc = mstat.PauseTotalNs
-	rsp.Threads = uint64(runtime.NumGoroutine())
-	return nil
-}

debug/log/default.go

@@ -0,0 +1,114 @@
+package log
+
+import (
+	"fmt"
+	golog "log"
+
+	"github.com/micro/go-micro/debug/buffer"
+)
+
+var (
+	// DefaultSize of the logger buffer
+	DefaultSize = 1000
+)
+
+// defaultLog is default micro log
+type defaultLog struct {
+	*buffer.Buffer
+}
+
+// NewLog returns default Logger with
+func NewLog(opts ...Option) Log {
+	// get default options
+	options := DefaultOptions()
+
+	// apply requested options
+	for _, o := range opts {
+		o(&options)
+	}
+
+	return &defaultLog{
+		Buffer: buffer.New(options.Size),
+	}
+}
+
+// Write writes logs into logger
+func (l *defaultLog) Write(r Record) {
+	golog.Print(r.Value)
+	l.Buffer.Put(fmt.Sprint(r.Value))
+}
+
+// Read reads logs and returns them
+func (l *defaultLog) Read(opts ...ReadOption) []Record {
+	options := ReadOptions{}
+	// initialize the read options
+	for _, o := range opts {
+		o(&options)
+	}
+
+	var entries []*buffer.Entry
+	// if Since options ha sbeen specified we honor it
+	if !options.Since.IsZero() {
+		entries = l.Buffer.Since(options.Since)
+	}
+
+	// only if we specified valid count constraint
+	// do we end up doing some serious if-else kung-fu
+	// if since constraint has been provided
+	// we return *count* number of logs since the given timestamp;
+	// otherwise we return last count number of logs
+	if options.Count > 0 {
+		switch len(entries) > 0 {
+		case true:
+			// if we request fewer logs than what since constraint gives us
+			if options.Count < len(entries) {
+				entries = entries[0:options.Count]
+			}
+		default:
+			entries = l.Buffer.Get(options.Count)
+		}
+	}
+
+	records := make([]Record, 0, len(entries))
+	for _, entry := range entries {
+		record := Record{
+			Timestamp: entry.Timestamp,
+			Value:     entry.Value,
+		}
+		records = append(records, record)
+	}
+
+	return records
+}
+
+// Stream returns channel for reading log records
+func (l *defaultLog) Stream(stop chan bool) <-chan Record {
+	// get stream channel from ring buffer
+	stream := l.Buffer.Stream(stop)
+	// make a buffered channel
+	records := make(chan Record, 128)
+	// get last 10 records
+	last10 := l.Buffer.Get(10)
+
+	// stream the log records
+	go func() {
+		// first send last 10 records
+		for _, entry := range last10 {
+			records <- Record{
+				Timestamp: entry.Timestamp,
+				Value:     entry.Value,
+				Metadata:  make(map[string]string),
+			}
+		}
+		// now stream continuously
+		for entry := range stream {
+			records <- Record{
+				Timestamp: entry.Timestamp,
+				Value:     entry.Value,
+				Metadata:  make(map[string]string),
+			}
+		}
+	}()
+
+	return records
+}

debug/log/default_test.go

@@ -0,0 +1,32 @@
+package log
+
+import (
+	"reflect"
+	"testing"
+)
+
+func TestLogger(t *testing.T) {
+	// set size to some value
+	size := 100
+	// override the global logger
+	DefaultLog = NewLog(Size(size))
+	// make sure we have the right size of the logger ring buffer
+	if DefaultLog.(*defaultLog).Size() != size {
+		t.Errorf("expected buffer size: %d, got: %d", size, DefaultLog.(*defaultLog).Size())
+	}
+
+	// Log some cruft
+	Info("foobar")
+	// increase the log level
+	DefaultLevel = LevelDebug
+	Debugf("foo %s", "bar")
+
+	// Check if the logs are stored in the logger ring buffer
+	expected := []string{"foobar", "foo bar"}
+	entries := DefaultLog.Read(Count(len(expected)))
+	for i, entry := range entries {
+		if !reflect.DeepEqual(entry.Value, expected[i]) {
+			t.Errorf("expected %s, got %s", expected[i], entry.Value)
+		}
+	}
+}

debug/log/log.go

@@ -0,0 +1,179 @@
+// Package log provides debug logging
+package log
+
+import (
+	"fmt"
+	"os"
+	"time"
+)
+
+var (
+	// DefaultLog logger
+	DefaultLog = NewLog()
+	// DefaultLevel is default log level
+	DefaultLevel = LevelInfo
+	// prefix for all messages
+	prefix string
+)
+
+// Log is event log
+type Log interface {
+	// Read reads log entries from the logger
+	Read(...ReadOption) []Record
+	// Write writes records to log
+	Write(Record)
+	// Stream log records
+	Stream(chan bool) <-chan Record
+}
+
+// Record is log record entry
+type Record struct {
+	// Timestamp of logged event
+	Timestamp time.Time
+	// Value contains log entry
+	Value interface{}
+	// Metadata to enrich log record
+	Metadata map[string]string
+}
+
+// level is a log level
+type Level int
+
+const (
+	LevelFatal Level = iota
+	LevelError
+	LevelInfo
+	LevelWarn
+	LevelDebug
+	LevelTrace
+)
+
+func init() {
+	switch os.Getenv("MICRO_LOG_LEVEL") {
+	case "trace":
+		DefaultLevel = LevelTrace
+	case "debug":
+		DefaultLevel = LevelDebug
+	case "warn":
+		DefaultLevel = LevelWarn
+	case "info":
+		DefaultLevel = LevelInfo
+	case "error":
+		DefaultLevel = LevelError
+	case "fatal":
+		DefaultLevel = LevelFatal
+	}
+}
+
+func log(v ...interface{}) {
+	if len(prefix) > 0 {
+		DefaultLog.Write(Record{Value: fmt.Sprint(append([]interface{}{prefix, " "}, v...)...)})
+		return
+	}
+	DefaultLog.Write(Record{Value: fmt.Sprint(v...)})
+}
+
+func logf(format string, v ...interface{}) {
+	if len(prefix) > 0 {
+		format = prefix + " " + format
+	}
+	DefaultLog.Write(Record{Value: fmt.Sprintf(format, v...)})
+}
+
+// WithLevel logs with the level specified
+func WithLevel(l Level, v ...interface{}) {
+	if l > DefaultLevel {
+		return
+	}
+	log(v...)
+}
+
+// WithLevel logs with the level specified
+func WithLevelf(l Level, format string, v ...interface{}) {
+	if l > DefaultLevel {
+		return
+	}
+	logf(format, v...)
+}
+
+// Trace provides trace level logging
+func Trace(v ...interface{}) {
+	WithLevel(LevelTrace, v...)
+}
+
+// Tracef provides trace level logging
+func Tracef(format string, v ...interface{}) {
+	WithLevelf(LevelTrace, format, v...)
+}
+
+// Debug provides debug level logging
+func Debug(v ...interface{}) {
+	WithLevel(LevelDebug, v...)
+}
+
+// Debugf provides debug level logging
+func Debugf(format string, v ...interface{}) {
+	WithLevelf(LevelDebug, format, v...)
+}
+
+// Warn provides warn level logging
+func Warn(v ...interface{}) {
+	WithLevel(LevelWarn, v...)
+}
+
+// Warnf provides warn level logging
+func Warnf(format string, v ...interface{}) {
+	WithLevelf(LevelWarn, format, v...)
+}
+
+// Info provides info level logging
+func Info(v ...interface{}) {
+	WithLevel(LevelInfo, v...)
+}
+
+// Infof provides info level logging
+func Infof(format string, v ...interface{}) {
+	WithLevelf(LevelInfo, format, v...)
+}
+
+// Error provides warn level logging
+func Error(v ...interface{}) {
+	WithLevel(LevelError, v...)
+}
+
+// Errorf provides warn level logging
+func Errorf(format string, v ...interface{}) {
+	WithLevelf(LevelError, format, v...)
+}
+
+// Fatal logs with Log and then exits with os.Exit(1)
+func Fatal(v ...interface{}) {
+	WithLevel(LevelFatal, v...)
+	os.Exit(1)
+}
+
+// Fatalf logs with Logf and then exits with os.Exit(1)
+func Fatalf(format string, v ...interface{}) {
+	WithLevelf(LevelFatal, format, v...)
+	os.Exit(1)
+}
+
+// SetLevel sets the log level
+func SetLevel(l Level) {
+	DefaultLevel = l
+}
+
+// GetLevel returns the current level
+func GetLevel() Level {
+	return DefaultLevel
+}
+
+// Set a prefix for the logger
+func SetPrefix(p string) {
+	prefix = p
+}
+
+// Set service name
+func Name(name string) {
+	prefix = fmt.Sprintf("[%s]", name)
+}

debug/log/options.go

@@ -0,0 +1,60 @@
+package log
+
+import "time"
+
+// Option used by the logger
+type Option func(*Options)
+
+// Options are logger options
+type Options struct {
+	// Size is the size of ring buffer
+	Size int
+}
+
+// Size sets the size of the ring buffer
+func Size(s int) Option {
+	return func(o *Options) {
+		o.Size = s
+	}
+}
+
+// DefaultOptions returns default options
+func DefaultOptions() Options {
+	return Options{
+		Size: DefaultSize,
+	}
+}
+
+// ReadOptions for querying the logs
+type ReadOptions struct {
+	// Since what time in past to return the logs
+	Since time.Time
+	// Count specifies number of logs to return
+	Count int
+	// Stream requests continuous log stream
+	Stream bool
+}
+
+// ReadOption used for reading the logs
+type ReadOption func(*ReadOptions)
+
+// Since sets the time since which to return the log records
+func Since(s time.Time) ReadOption {
+	return func(o *ReadOptions) {
+		o.Since = s
+	}
+}
+
+// Count sets the number of log records to return
+func Count(c int) ReadOption {
+	return func(o *ReadOptions) {
+		o.Count = c
+	}
+}
+
+// Stream requests continuous log stream
+func Stream(s bool) ReadOption {
+	return func(o *ReadOptions) {
+		o.Stream = s
+	}
+}

debug/profile/pprof/pprof.go

@@ -0,0 +1,118 @@
+// Package pprof provides a pprof profiler
+package pprof
+
+import (
+	"os"
+	"path/filepath"
+	"runtime"
+	"runtime/pprof"
+	"sync"
+	"time"
+
+	"github.com/micro/go-micro/debug/profile"
+)
+
+type profiler struct {
+	opts profile.Options
+
+	sync.Mutex
+	running bool
+	exit    chan bool
+
+	// where the cpu profile is written
+	cpuFile *os.File
+	// where the mem profile is written
+	memFile *os.File
+}
+
+func (p *profiler) writeHeap(f *os.File) {
+	defer f.Close()
+
+	t := time.NewTicker(time.Second * 30)
+	defer t.Stop()
+
+	for {
+		select {
+		case <-t.C:
+			runtime.GC()
+			pprof.WriteHeapProfile(f)
+		case <-p.exit:
+			return
+		}
+	}
+}
+
+func (p *profiler) Start() error {
+	p.Lock()
+	defer p.Unlock()
+
+	if p.running {
+		return nil
+	}
+
+	// create exit channel
+	p.exit = make(chan bool)
+
+	cpuFile := filepath.Join("/tmp", "cpu.pprof")
+	memFile := filepath.Join("/tmp", "mem.pprof")
+
+	if len(p.opts.Name) > 0 {
+		cpuFile = filepath.Join("/tmp", p.opts.Name+".cpu.pprof")
+		memFile = filepath.Join("/tmp", p.opts.Name+".mem.pprof")
+	}
+
+	f1, err := os.Create(cpuFile)
+	if err != nil {
+		return err
+	}
+
+	f2, err := os.Create(memFile)
+	if err != nil {
+		return err
+	}
+
+	// start cpu profiling
+	if err := pprof.StartCPUProfile(f1); err != nil {
+		return err
+	}
+
+	// write the heap periodically
+	go p.writeHeap(f2)
+
+	// set cpu file
+	p.cpuFile = f1
+	// set mem file
+	p.memFile = f2
+
+	p.running = true
+
+	return nil
+}
+
+func (p *profiler) Stop() error {
+	p.Lock()
+	defer p.Unlock()
+
+	select {
+	case <-p.exit:
+		return nil
+	default:
+		close(p.exit)
+		pprof.StopCPUProfile()
+		p.cpuFile.Close()
+		p.running = false
+		p.cpuFile = nil
+		p.memFile = nil
+		return nil
+	}
+}
+
+func NewProfile(opts ...profile.Option) profile.Profile {
+	var options profile.Options
+	for _, o := range opts {
+		o(&options)
+	}
+	p := new(profiler)
+	p.opts = options
+	return p
+}

debug/profile/profile.go

@@ -0,0 +1,23 @@
+// Package profile is for profilers
+package profile
+
+type Profile interface {
+	// Start the profiler
+	Start() error
+	// Stop the profiler
+	Stop() error
+}
+
+type Options struct {
+	// Name to use for the profile
+	Name string
+}
+
+type Option func(o *Options)
+
+// Name of the profile
+func Name(n string) Option {
+	return func(o *Options) {
+		o.Name = n
+	}
+}

debug/proto/debug.pb.go

@@ -1,336 +0,0 @@
-// Code generated by protoc-gen-go. DO NOT EDIT.
-// source: micro/go-micro/debug/proto/debug.proto
-
-package debug
-
-import (
-	context "context"
-	fmt "fmt"
-	proto "github.com/golang/protobuf/proto"
-	grpc "google.golang.org/grpc"
-	math "math"
-)
-
-// Reference imports to suppress errors if they are not otherwise used.
-var _ = proto.Marshal
-var _ = fmt.Errorf
-var _ = math.Inf
-
-// This is a compile-time assertion to ensure that this generated file
-// is compatible with the proto package it is being compiled against.
-// A compilation error at this line likely means your copy of the
-// proto package needs to be updated.
-const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package
-
-type HealthRequest struct {
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
-}
-
-func (m *HealthRequest) Reset()         { *m = HealthRequest{} }
-func (m *HealthRequest) String() string { return proto.CompactTextString(m) }
-func (*HealthRequest) ProtoMessage()    {}
-func (*HealthRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_f25415e61bccfa1f, []int{0}
-}
-
-func (m *HealthRequest) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_HealthRequest.Unmarshal(m, b)
-}
-func (m *HealthRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_HealthRequest.Marshal(b, m, deterministic)
-}
-func (m *HealthRequest) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_HealthRequest.Merge(m, src)
-}
-func (m *HealthRequest) XXX_Size() int {
-	return xxx_messageInfo_HealthRequest.Size(m)
-}
-func (m *HealthRequest) XXX_DiscardUnknown() {
-	xxx_messageInfo_HealthRequest.DiscardUnknown(m)
-}
-
-var xxx_messageInfo_HealthRequest proto.InternalMessageInfo
-
-type HealthResponse struct {
-	// default: ok
-	Status               string   `protobuf:"bytes,1,opt,name=status,proto3" json:"status,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
-}
-
-func (m *HealthResponse) Reset()         { *m = HealthResponse{} }
-func (m *HealthResponse) String() string { return proto.CompactTextString(m) }
-func (*HealthResponse) ProtoMessage()    {}
-func (*HealthResponse) Descriptor() ([]byte, []int) {
-	return fileDescriptor_f25415e61bccfa1f, []int{1}
-}
-
-func (m *HealthResponse) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_HealthResponse.Unmarshal(m, b)
-}
-func (m *HealthResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_HealthResponse.Marshal(b, m, deterministic)
-}
-func (m *HealthResponse) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_HealthResponse.Merge(m, src)
-}
-func (m *HealthResponse) XXX_Size() int {
-	return xxx_messageInfo_HealthResponse.Size(m)
-}
-func (m *HealthResponse) XXX_DiscardUnknown() {
-	xxx_messageInfo_HealthResponse.DiscardUnknown(m)
-}
-
-var xxx_messageInfo_HealthResponse proto.InternalMessageInfo
-
-func (m *HealthResponse) GetStatus() string {
-	if m != nil {
-		return m.Status
-	}
-	return ""
-}
-
-type StatsRequest struct {
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
-}
-
-func (m *StatsRequest) Reset()         { *m = StatsRequest{} }
-func (m *StatsRequest) String() string { return proto.CompactTextString(m) }
-func (*StatsRequest) ProtoMessage()    {}
-func (*StatsRequest) Descriptor() ([]byte, []int) {
-	return fileDescriptor_f25415e61bccfa1f, []int{2}
-}
-
-func (m *StatsRequest) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_StatsRequest.Unmarshal(m, b)
-}
-func (m *StatsRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_StatsRequest.Marshal(b, m, deterministic)
-}
-func (m *StatsRequest) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_StatsRequest.Merge(m, src)
-}
-func (m *StatsRequest) XXX_Size() int {
-	return xxx_messageInfo_StatsRequest.Size(m)
-}
-func (m *StatsRequest) XXX_DiscardUnknown() {
-	xxx_messageInfo_StatsRequest.DiscardUnknown(m)
-}
-
-var xxx_messageInfo_StatsRequest proto.InternalMessageInfo
-
-type StatsResponse struct {
-	// unix timestamp
-	Started uint64 `protobuf:"varint,1,opt,name=started,proto3" json:"started,omitempty"`
-	// in seconds
-	Uptime uint64 `protobuf:"varint,2,opt,name=uptime,proto3" json:"uptime,omitempty"`
-	// in bytes
-	Memory uint64 `protobuf:"varint,3,opt,name=memory,proto3" json:"memory,omitempty"`
-	// num threads
-	Threads uint64 `protobuf:"varint,4,opt,name=threads,proto3" json:"threads,omitempty"`
-	// total gc in nanoseconds
-	Gc                   uint64   `protobuf:"varint,5,opt,name=gc,proto3" json:"gc,omitempty"`
-	XXX_NoUnkeyedLiteral struct{} `json:"-"`
-	XXX_unrecognized     []byte   `json:"-"`
-	XXX_sizecache        int32    `json:"-"`
-}
-
-func (m *StatsResponse) Reset()         { *m = StatsResponse{} }
-func (m *StatsResponse) String() string { return proto.CompactTextString(m) }
-func (*StatsResponse) ProtoMessage()    {}
-func (*StatsResponse) Descriptor() ([]byte, []int) {
-	return fileDescriptor_f25415e61bccfa1f, []int{3}
-}
-
-func (m *StatsResponse) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_StatsResponse.Unmarshal(m, b)
-}
-func (m *StatsResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_StatsResponse.Marshal(b, m, deterministic)
-}
-func (m *StatsResponse) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_StatsResponse.Merge(m, src)
-}
-func (m *StatsResponse) XXX_Size() int {
-	return xxx_messageInfo_StatsResponse.Size(m)
-}
-func (m *StatsResponse) XXX_DiscardUnknown() {
-	xxx_messageInfo_StatsResponse.DiscardUnknown(m)
-}
-
-var xxx_messageInfo_StatsResponse proto.InternalMessageInfo
-
-func (m *StatsResponse) GetStarted() uint64 {
-	if m != nil {
-		return m.Started
-	}
-	return 0
-}
-
-func (m *StatsResponse) GetUptime() uint64 {
-	if m != nil {
-		return m.Uptime
-	}
-	return 0
-}
-
-func (m *StatsResponse) GetMemory() uint64 {
-	if m != nil {
-		return m.Memory
-	}
-	return 0
-}
-
-func (m *StatsResponse) GetThreads() uint64 {
-	if m != nil {
-		return m.Threads
-	}
-	return 0
-}
-
-func (m *StatsResponse) GetGc() uint64 {
-	if m != nil {
-		return m.Gc
-	}
-	return 0
-}
-
-func init() {
-	proto.RegisterType((*HealthRequest)(nil), "HealthRequest")
-	proto.RegisterType((*HealthResponse)(nil), "HealthResponse")
-	proto.RegisterType((*StatsRequest)(nil), "StatsRequest")
-	proto.RegisterType((*StatsResponse)(nil), "StatsResponse")
-}
-
-func init() {
-	proto.RegisterFile("micro/go-micro/debug/proto/debug.proto", fileDescriptor_f25415e61bccfa1f)
-}
-
-var fileDescriptor_f25415e61bccfa1f = []byte{
-	// 230 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x54, 0x90, 0x41, 0x4b, 0xc4, 0x30,
-	0x10, 0x85, 0xb7, 0x75, 0x5b, 0x71, 0xb0, 0x59, 0xc8, 0x41, 0xc2, 0x9e, 0x24, 0x07, 0x29, 0x88,
-	0x59, 0xd0, 0xbf, 0xe0, 0xc1, 0x73, 0xbd, 0x0b, 0xd9, 0x76, 0xe8, 0x16, 0xac, 0xa9, 0x99, 0xe9,
-	0xc1, 0xb3, 0x7f, 0x5c, 0x9a, 0xa4, 0x60, 0x6f, 0xef, 0xbd, 0xf0, 0x1e, 0xf9, 0x06, 0x1e, 0xc6,
-	0xa1, 0xf5, 0xee, 0xd4, 0xbb, 0xa7, 0x28, 0x3a, 0x3c, 0xcf, 0xfd, 0x69, 0xf2, 0x8e, 0x93, 0x36,
-	0x41, 0xeb, 0x03, 0x54, 0x6f, 0x68, 0x3f, 0xf9, 0xd2, 0xe0, 0xf7, 0x8c, 0xc4, 0xba, 0x06, 0xb1,
-	0x06, 0x34, 0xb9, 0x2f, 0x42, 0x79, 0x07, 0x25, 0xb1, 0xe5, 0x99, 0x54, 0x76, 0x9f, 0xd5, 0x37,
-	0x4d, 0x72, 0x5a, 0xc0, 0xed, 0x3b, 0x5b, 0xa6, 0xb5, 0xf9, 0x9b, 0x41, 0x95, 0x82, 0xd4, 0x54,
-	0x70, 0x4d, 0x6c, 0x3d, 0x63, 0x17, 0xaa, 0xfb, 0x66, 0xb5, 0xcb, 0xe6, 0x3c, 0xf1, 0x30, 0xa2,
-	0xca, 0xc3, 0x43, 0x72, 0x4b, 0x3e, 0xe2, 0xe8, 0xfc, 0x8f, 0xba, 0x8a, 0x79, 0x74, 0xcb, 0x12,
-	0x5f, 0x3c, 0xda, 0x8e, 0xd4, 0x3e, 0x2e, 0x25, 0x2b, 0x05, 0xe4, 0x7d, 0xab, 0x8a, 0x10, 0xe6,
-	0x7d, 0xfb, 0xfc, 0x01, 0xc5, 0xeb, 0xc2, 0x27, 0x1f, 0xa1, 0x8c, 0x20, 0x52, 0x98, 0x0d, 0xe2,
-	0xf1, 0x60, 0xb6, 0x84, 0x7a, 0x27, 0x6b, 0x28, 0xc2, 0xd7, 0x65, 0x65, 0xfe, 0x33, 0x1d, 0x85,
-	0xd9, 0x10, 0xe9, 0xdd, 0xb9, 0x0c, 0x77, 0x7b, 0xf9, 0x0b, 0x00, 0x00, 0xff, 0xff, 0xfe, 0xb9,
-	0x5f, 0xf7, 0x61, 0x01, 0x00, 0x00,
-}
-
-// Reference imports to suppress errors if they are not otherwise used.
-var _ context.Context
-var _ grpc.ClientConn
-
-// This is a compile-time assertion to ensure that this generated file
-// is compatible with the grpc package it is being compiled against.
-const _ = grpc.SupportPackageIsVersion4
-
-// DebugClient is the client API for Debug service.
-//
-// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream.
-type DebugClient interface {
-	Health(ctx context.Context, in *HealthRequest, opts ...grpc.CallOption) (*HealthResponse, error)
-	Stats(ctx context.Context, in *StatsRequest, opts ...grpc.CallOption) (*StatsResponse, error)
-}
-
-type debugClient struct {
-	cc *grpc.ClientConn
-}
-
-func NewDebugClient(cc *grpc.ClientConn) DebugClient {
-	return &debugClient{cc}
-}
-
-func (c *debugClient) Health(ctx context.Context, in *HealthRequest, opts ...grpc.CallOption) (*HealthResponse, error) {
-	out := new(HealthResponse)
-	err := c.cc.Invoke(ctx, "/Debug/Health", in, out, opts...)
-	if err != nil {
-		return nil, err
-	}
-	return out, nil
-}
-
-func (c *debugClient) Stats(ctx context.Context, in *StatsRequest, opts ...grpc.CallOption) (*StatsResponse, error) {
-	out := new(StatsResponse)
-	err := c.cc.Invoke(ctx, "/Debug/Stats", in, out, opts...)
-	if err != nil {
-		return nil, err
-	}
-	return out, nil
-}
-
-// DebugServer is the server API for Debug service.
-type DebugServer interface {
-	Health(context.Context, *HealthRequest) (*HealthResponse, error)
-	Stats(context.Context, *StatsRequest) (*StatsResponse, error)
-}
-
-func RegisterDebugServer(s *grpc.Server, srv DebugServer) {
-	s.RegisterService(&_Debug_serviceDesc, srv)
-}
-
-func _Debug_Health_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
-	in := new(HealthRequest)
-	if err := dec(in); err != nil {
-		return nil, err
-	}
-	if interceptor == nil {
-		return srv.(DebugServer).Health(ctx, in)
-	}
-	info := &grpc.UnaryServerInfo{
-		Server:     srv,
-		FullMethod: "/Debug/Health",
-	}
-	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
-		return srv.(DebugServer).Health(ctx, req.(*HealthRequest))
-	}
-	return interceptor(ctx, in, info, handler)
-}
-
-func _Debug_Stats_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
-	in := new(StatsRequest)
-	if err := dec(in); err != nil {
-		return nil, err
-	}
-	if interceptor == nil {
-		return srv.(DebugServer).Stats(ctx, in)
-	}
-	info := &grpc.UnaryServerInfo{
-		Server:     srv,
-		FullMethod: "/Debug/Stats",
-	}
-	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
-		return srv.(DebugServer).Stats(ctx, req.(*StatsRequest))
-	}
-	return interceptor(ctx, in, info, handler)
-}
-
-var _Debug_serviceDesc = grpc.ServiceDesc{
-	ServiceName: "Debug",
-	HandlerType: (*DebugServer)(nil),
-	Methods: []grpc.MethodDesc{
-		{
-			MethodName: "Health",
-			Handler:    _Debug_Health_Handler,
-		},
-		{
-			MethodName: "Stats",
-			Handler:    _Debug_Stats_Handler,
-		},
-	},
-	Streams:  []grpc.StreamDesc{},
-	Metadata: "micro/go-micro/debug/proto/debug.proto",
-}

debug/proto/debug.proto

@@ -1,30 +0,0 @@
-syntax = "proto3";
-
-service Debug {
- 	rpc Health(HealthRequest) returns (HealthResponse) {}
- 	rpc Stats(StatsRequest) returns (StatsResponse) {}
-}
-
-message HealthRequest {
-}
-
-message HealthResponse {
-	// default: ok
-	string status = 1;
-}
-
-message StatsRequest {
-}
-
-message StatsResponse {
-	// unix timestamp
-	uint64 started = 1;
-	// in seconds
-	uint64 uptime = 2;
-	// in bytes
-	uint64 memory = 3;
-	// num threads
-	uint64 threads = 4;
-	// total gc in nanoseconds
-	uint64 gc = 5;
-}

debug/service/handler/debug.go

@@ -0,0 +1,116 @@
+// Pacjage handler implements service debug handler
+package handler
+
+import (
+	"context"
+	"runtime"
+	"time"
+
+	"github.com/micro/go-micro/debug/log"
+	proto "github.com/micro/go-micro/debug/service/proto"
+	"github.com/micro/go-micro/server"
+)
+
+var (
+	// DefaultHandler is default debug handler
+	DefaultHandler = newDebug()
+)
+
+type Debug struct {
+	started int64
+	proto.DebugHandler
+	log log.Log
+}
+
+func newDebug() *Debug {
+	return &Debug{
+		started: time.Now().Unix(),
+		log:     log.DefaultLog,
+	}
+}
+
+func (d *Debug) Health(ctx context.Context, req *proto.HealthRequest, rsp *proto.HealthResponse) error {
+	rsp.Status = "ok"
+	return nil
+}
+
+func (d *Debug) Stats(ctx context.Context, req *proto.StatsRequest, rsp *proto.StatsResponse) error {
+	var mstat runtime.MemStats
+	runtime.ReadMemStats(&mstat)
+
+	rsp.Timestamp = uint64(time.Now().Unix())
+	rsp.Started = uint64(d.started)
+	rsp.Uptime = uint64(time.Now().Unix() - d.started)
+	rsp.Memory = mstat.Alloc
+	rsp.Gc = mstat.PauseTotalNs
+	rsp.Threads = uint64(runtime.NumGoroutine())
+	return nil
+}
+
+func (d *Debug) Log(ctx context.Context, stream server.Stream) error {
+	req := new(proto.LogRequest)
+	if err := stream.Recv(req); err != nil {
+		return err
+	}
+
+	var options []log.ReadOption
+
+	since := time.Unix(req.Since, 0)
+	if !since.IsZero() {
+		options = append(options, log.Since(since))
+	}
+
+	count := int(req.Count)
+	if count > 0 {
+		options = append(options, log.Count(count))
+	}
+
+	if req.Stream {
+		stop := make(chan bool)
+		defer close(stop)
+
+		// TODO: we need to figure out how to close ithe log stream
+		// It seems like when a client disconnects,
+		// the connection stays open until some timeout expires
+		// or something like that; that means the map of streams
+		// might end up leaking memory if not cleaned up properly
+		records := d.log.Stream(stop)
+		for record := range records {
+			if err := d.sendRecord(record, stream); err != nil {
+				return err
+			}
+		}
+		// done streaming, return
+		return nil
+	}
+
+	// get the log records
+	records := d.log.Read(options...)
+	// send all the logs downstream
+	for _, record := range records {
+		if err := d.sendRecord(record, stream); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func (d *Debug) sendRecord(record log.Record, stream server.Stream) error {
+	metadata := make(map[string]string)
+	for k, v := range record.Metadata {
+		metadata[k] = v
+	}
+
+	pbRecord := &proto.Record{
+		Timestamp: record.Timestamp.Unix(),
+		Value:     record.Value.(string),
+		Metadata:  metadata,
+	}
+
+	if err := stream.Send(pbRecord); err != nil {
+		return err
+	}
+
+	return nil
+}

debug/service/proto/debug.pb.go

@@ -0,0 +1,398 @@
+// Code generated by protoc-gen-go. DO NOT EDIT.
+// source: micro/go-micro/debug/service/proto/debug.proto
+
+package debug
+
+import (
+	fmt "fmt"
+	proto "github.com/golang/protobuf/proto"
+	math "math"
+)
+
+// Reference imports to suppress errors if they are not otherwise used.
+var _ = proto.Marshal
+var _ = fmt.Errorf
+var _ = math.Inf
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the proto package it is being compiled against.
+// A compilation error at this line likely means your copy of the
+// proto package needs to be updated.
+const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package
+
+type HealthRequest struct {
+	// optional service name
+	Service              string   `protobuf:"bytes,1,opt,name=service,proto3" json:"service,omitempty"`
+	XXX_NoUnkeyedLiteral struct{} `json:"-"`
+	XXX_unrecognized     []byte   `json:"-"`
+	XXX_sizecache        int32    `json:"-"`
+}
+
+func (m *HealthRequest) Reset()         { *m = HealthRequest{} }
+func (m *HealthRequest) String() string { return proto.CompactTextString(m) }
+func (*HealthRequest) ProtoMessage()    {}
+func (*HealthRequest) Descriptor() ([]byte, []int) {
+	return fileDescriptor_dea322649cde1ef2, []int{0}
+}
+
+func (m *HealthRequest) XXX_Unmarshal(b []byte) error {
+	return xxx_messageInfo_HealthRequest.Unmarshal(m, b)
+}
+func (m *HealthRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	return xxx_messageInfo_HealthRequest.Marshal(b, m, deterministic)
+}
+func (m *HealthRequest) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_HealthRequest.Merge(m, src)
+}
+func (m *HealthRequest) XXX_Size() int {
+	return xxx_messageInfo_HealthRequest.Size(m)
+}
+func (m *HealthRequest) XXX_DiscardUnknown() {
+	xxx_messageInfo_HealthRequest.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_HealthRequest proto.InternalMessageInfo
+
+func (m *HealthRequest) GetService() string {
+	if m != nil {
+		return m.Service
+	}
+	return ""
+}
+
+type HealthResponse struct {
+	// default: ok
+	Status               string   `protobuf:"bytes,1,opt,name=status,proto3" json:"status,omitempty"`
+	XXX_NoUnkeyedLiteral struct{} `json:"-"`
+	XXX_unrecognized     []byte   `json:"-"`
+	XXX_sizecache        int32    `json:"-"`
+}
+
+func (m *HealthResponse) Reset()         { *m = HealthResponse{} }
+func (m *HealthResponse) String() string { return proto.CompactTextString(m) }
+func (*HealthResponse) ProtoMessage()    {}
+func (*HealthResponse) Descriptor() ([]byte, []int) {
+	return fileDescriptor_dea322649cde1ef2, []int{1}
+}
+
+func (m *HealthResponse) XXX_Unmarshal(b []byte) error {
+	return xxx_messageInfo_HealthResponse.Unmarshal(m, b)
+}
+func (m *HealthResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	return xxx_messageInfo_HealthResponse.Marshal(b, m, deterministic)
+}
+func (m *HealthResponse) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_HealthResponse.Merge(m, src)
+}
+func (m *HealthResponse) XXX_Size() int {
+	return xxx_messageInfo_HealthResponse.Size(m)
+}
+func (m *HealthResponse) XXX_DiscardUnknown() {
+	xxx_messageInfo_HealthResponse.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_HealthResponse proto.InternalMessageInfo
+
+func (m *HealthResponse) GetStatus() string {
+	if m != nil {
+		return m.Status
+	}
+	return ""
+}
+
+type StatsRequest struct {
+	// optional service name
+	Service              string   `protobuf:"bytes,1,opt,name=service,proto3" json:"service,omitempty"`
+	XXX_NoUnkeyedLiteral struct{} `json:"-"`
+	XXX_unrecognized     []byte   `json:"-"`
+	XXX_sizecache        int32    `json:"-"`
+}
+
+func (m *StatsRequest) Reset()         { *m = StatsRequest{} }
+func (m *StatsRequest) String() string { return proto.CompactTextString(m) }
+func (*StatsRequest) ProtoMessage()    {}
+func (*StatsRequest) Descriptor() ([]byte, []int) {
+	return fileDescriptor_dea322649cde1ef2, []int{2}
+}
+
+func (m *StatsRequest) XXX_Unmarshal(b []byte) error {
+	return xxx_messageInfo_StatsRequest.Unmarshal(m, b)
+}
+func (m *StatsRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	return xxx_messageInfo_StatsRequest.Marshal(b, m, deterministic)
+}
+func (m *StatsRequest) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_StatsRequest.Merge(m, src)
+}
+func (m *StatsRequest) XXX_Size() int {
+	return xxx_messageInfo_StatsRequest.Size(m)
+}
+func (m *StatsRequest) XXX_DiscardUnknown() {
+	xxx_messageInfo_StatsRequest.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_StatsRequest proto.InternalMessageInfo
+
+func (m *StatsRequest) GetService() string {
+	if m != nil {
+		return m.Service
+	}
+	return ""
+}
+
+type StatsResponse struct {
+	// timestamp of recording
+	Timestamp uint64 `protobuf:"varint,1,opt,name=timestamp,proto3" json:"timestamp,omitempty"`
+	// unix timestamp
+	Started uint64 `protobuf:"varint,2,opt,name=started,proto3" json:"started,omitempty"`
+	// in seconds
+	Uptime uint64 `protobuf:"varint,3,opt,name=uptime,proto3" json:"uptime,omitempty"`
+	// in bytes
+	Memory uint64 `protobuf:"varint,4,opt,name=memory,proto3" json:"memory,omitempty"`
+	// num threads
+	Threads uint64 `protobuf:"varint,5,opt,name=threads,proto3" json:"threads,omitempty"`
+	// total gc in nanoseconds
+	Gc                   uint64   `protobuf:"varint,6,opt,name=gc,proto3" json:"gc,omitempty"`
+	XXX_NoUnkeyedLiteral struct{} `json:"-"`
+	XXX_unrecognized     []byte   `json:"-"`
+	XXX_sizecache        int32    `json:"-"`
+}
+
+func (m *StatsResponse) Reset()         { *m = StatsResponse{} }
+func (m *StatsResponse) String() string { return proto.CompactTextString(m) }
+func (*StatsResponse) ProtoMessage()    {}
+func (*StatsResponse) Descriptor() ([]byte, []int) {
+	return fileDescriptor_dea322649cde1ef2, []int{3}
+}
+
+func (m *StatsResponse) XXX_Unmarshal(b []byte) error {
+	return xxx_messageInfo_StatsResponse.Unmarshal(m, b)
+}
+func (m *StatsResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	return xxx_messageInfo_StatsResponse.Marshal(b, m, deterministic)
+}
+func (m *StatsResponse) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_StatsResponse.Merge(m, src)
+}
+func (m *StatsResponse) XXX_Size() int {
+	return xxx_messageInfo_StatsResponse.Size(m)
+}
+func (m *StatsResponse) XXX_DiscardUnknown() {
+	xxx_messageInfo_StatsResponse.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_StatsResponse proto.InternalMessageInfo
+
+func (m *StatsResponse) GetTimestamp() uint64 {
+	if m != nil {
+		return m.Timestamp
+	}
+	return 0
+}
+
+func (m *StatsResponse) GetStarted() uint64 {
+	if m != nil {
+		return m.Started
+	}
+	return 0
+}
+
+func (m *StatsResponse) GetUptime() uint64 {
+	if m != nil {
+		return m.Uptime
+	}
+	return 0
+}
+
+func (m *StatsResponse) GetMemory() uint64 {
+	if m != nil {
+		return m.Memory
+	}
+	return 0
+}
+
+func (m *StatsResponse) GetThreads() uint64 {
+	if m != nil {
+		return m.Threads
+	}
+	return 0
+}
+
+func (m *StatsResponse) GetGc() uint64 {
+	if m != nil {
+		return m.Gc
+	}
+	return 0
+}
+
+// LogRequest requests service logs
+type LogRequest struct {
+	// service to request logs for
+	Service string `protobuf:"bytes,1,opt,name=service,proto3" json:"service,omitempty"`
+	// stream records continuously
+	Stream bool `protobuf:"varint,2,opt,name=stream,proto3" json:"stream,omitempty"`
+	// count of records to request
+	Count int64 `protobuf:"varint,3,opt,name=count,proto3" json:"count,omitempty"`
+	// relative time in seconds
+	// before the current time
+	// from which to show logs
+	Since                int64    `protobuf:"varint,4,opt,name=since,proto3" json:"since,omitempty"`
+	XXX_NoUnkeyedLiteral struct{} `json:"-"`
+	XXX_unrecognized     []byte   `json:"-"`
+	XXX_sizecache        int32    `json:"-"`
+}
+
+func (m *LogRequest) Reset()         { *m = LogRequest{} }
+func (m *LogRequest) String() string { return proto.CompactTextString(m) }
+func (*LogRequest) ProtoMessage()    {}
+func (*LogRequest) Descriptor() ([]byte, []int) {
+	return fileDescriptor_dea322649cde1ef2, []int{4}
+}
+
+func (m *LogRequest) XXX_Unmarshal(b []byte) error {
+	return xxx_messageInfo_LogRequest.Unmarshal(m, b)
+}
+func (m *LogRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	return xxx_messageInfo_LogRequest.Marshal(b, m, deterministic)
+}
+func (m *LogRequest) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_LogRequest.Merge(m, src)
+}
+func (m *LogRequest) XXX_Size() int {
+	return xxx_messageInfo_LogRequest.Size(m)
+}
+func (m *LogRequest) XXX_DiscardUnknown() {
+	xxx_messageInfo_LogRequest.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_LogRequest proto.InternalMessageInfo
+
+func (m *LogRequest) GetService() string {
+	if m != nil {
+		return m.Service
+	}
+	return ""
+}
+
+func (m *LogRequest) GetStream() bool {
+	if m != nil {
+		return m.Stream
+	}
+	return false
+}
+
+func (m *LogRequest) GetCount() int64 {
+	if m != nil {
+		return m.Count
+	}
+	return 0
+}
+
+func (m *LogRequest) GetSince() int64 {
+	if m != nil {
+		return m.Since
+	}
+	return 0
+}
+
+// Record is service log record
+type Record struct {
+	// timestamp of log record
+	Timestamp int64 `protobuf:"varint,1,opt,name=timestamp,proto3" json:"timestamp,omitempty"`
+	// record value
+	Value string `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"`
+	// record metadata
+	Metadata             map[string]string `protobuf:"bytes,3,rep,name=metadata,proto3" json:"metadata,omitempty" protobuf_key:"bytes,1,opt,name=key,proto3" protobuf_val:"bytes,2,opt,name=value,proto3"`
+	XXX_NoUnkeyedLiteral struct{}          `json:"-"`
+	XXX_unrecognized     []byte            `json:"-"`
+	XXX_sizecache        int32             `json:"-"`
+}
+
+func (m *Record) Reset()         { *m = Record{} }
+func (m *Record) String() string { return proto.CompactTextString(m) }
+func (*Record) ProtoMessage()    {}
+func (*Record) Descriptor() ([]byte, []int) {
+	return fileDescriptor_dea322649cde1ef2, []int{5}
+}
+
+func (m *Record) XXX_Unmarshal(b []byte) error {
+	return xxx_messageInfo_Record.Unmarshal(m, b)
+}
+func (m *Record) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	return xxx_messageInfo_Record.Marshal(b, m, deterministic)
+}
+func (m *Record) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_Record.Merge(m, src)
+}
+func (m *Record) XXX_Size() int {
+	return xxx_messageInfo_Record.Size(m)
+}
+func (m *Record) XXX_DiscardUnknown() {
+	xxx_messageInfo_Record.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_Record proto.InternalMessageInfo
+
+func (m *Record) GetTimestamp() int64 {
+	if m != nil {
+		return m.Timestamp
+	}
+	return 0
+}
+
+func (m *Record) GetValue() string {
+	if m != nil {
+		return m.Value
+	}
+	return ""
+}
+
+func (m *Record) GetMetadata() map[string]string {
+	if m != nil {
+		return m.Metadata
+	}
+	return nil
+}
+
+func init() {
+	proto.RegisterType((*HealthRequest)(nil), "HealthRequest")
+	proto.RegisterType((*HealthResponse)(nil), "HealthResponse")
+	proto.RegisterType((*StatsRequest)(nil), "StatsRequest")
+	proto.RegisterType((*StatsResponse)(nil), "StatsResponse")
+	proto.RegisterType((*LogRequest)(nil), "LogRequest")
+	proto.RegisterType((*Record)(nil), "Record")
+	proto.RegisterMapType((map[string]string)(nil), "Record.MetadataEntry")
+}
+
+func init() {
+	proto.RegisterFile("micro/go-micro/debug/service/proto/debug.proto", fileDescriptor_dea322649cde1ef2)
+}
+
+var fileDescriptor_dea322649cde1ef2 = []byte{
+	// 399 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x84, 0x52, 0xc1, 0x6e, 0xd4, 0x30,
+	0x10, 0xdd, 0xac, 0x9b, 0xb4, 0x3b, 0x65, 0x17, 0x64, 0x15, 0x64, 0xad, 0x90, 0xa8, 0x7c, 0x0a,
+	0x42, 0x78, 0xa1, 0x5c, 0x10, 0x5c, 0x41, 0xe2, 0x50, 0x2e, 0xe6, 0x0b, 0xdc, 0x64, 0x94, 0x2e,
+	0x34, 0x71, 0xb0, 0x27, 0x95, 0xf6, 0xc4, 0xb7, 0x70, 0xe7, 0x23, 0x51, 0x6c, 0x2f, 0x6d, 0x44,
+	0xa5, 0xbd, 0xcd, 0x7b, 0xf3, 0xf4, 0x3c, 0x33, 0x7e, 0xa0, 0xda, 0x6d, 0xe5, 0xec, 0xa6, 0xb1,
+	0xaf, 0x63, 0x51, 0xe3, 0xd5, 0xd0, 0x6c, 0x3c, 0xba, 0xdb, 0x6d, 0x85, 0x9b, 0xde, 0x59, 0x4a,
+	0x9c, 0x0a, 0xb5, 0x7c, 0x09, 0xcb, 0x2f, 0x68, 0x6e, 0xe8, 0x5a, 0xe3, 0xcf, 0x01, 0x3d, 0x71,
+	0x01, 0xc7, 0x49, 0x2d, 0xb2, 0xf3, 0xac, 0x5c, 0xe8, 0x3d, 0x94, 0x25, 0xac, 0xf6, 0x52, 0xdf,
+	0xdb, 0xce, 0x23, 0x7f, 0x06, 0x85, 0x27, 0x43, 0x83, 0x4f, 0xd2, 0x84, 0x64, 0x09, 0x8f, 0xbe,
+	0x91, 0x21, 0x7f, 0xd8, 0xf3, 0x77, 0x06, 0xcb, 0x24, 0x4d, 0x9e, 0xcf, 0x61, 0x41, 0xdb, 0x16,
+	0x3d, 0x99, 0xb6, 0x0f, 0xea, 0x23, 0x7d, 0x47, 0x04, 0x27, 0x32, 0x8e, 0xb0, 0x16, 0xf3, 0xd0,
+	0xdb, 0xc3, 0x71, 0x96, 0xa1, 0x1f, 0x85, 0x82, 0x85, 0x46, 0x42, 0x23, 0xdf, 0x62, 0x6b, 0xdd,
+	0x4e, 0x1c, 0x45, 0x3e, 0xa2, 0xd1, 0x89, 0xae, 0x1d, 0x9a, 0xda, 0x8b, 0x3c, 0x3a, 0x25, 0xc8,
+	0x57, 0x30, 0x6f, 0x2a, 0x51, 0x04, 0x72, 0xde, 0x54, 0xf2, 0x3b, 0xc0, 0xa5, 0x6d, 0x0e, 0xee,
+	0x12, 0xaf, 0xe1, 0xd0, 0xb4, 0x61, 0xb4, 0x13, 0x9d, 0x10, 0x3f, 0x83, 0xbc, 0xb2, 0x43, 0x47,
+	0x61, 0x30, 0xa6, 0x23, 0x18, 0x59, 0xbf, 0xed, 0x2a, 0x0c, 0x63, 0x31, 0x1d, 0x81, 0xfc, 0x93,
+	0x41, 0xa1, 0xb1, 0xb2, 0xae, 0xfe, 0xff, 0x10, 0xec, 0xfe, 0x21, 0xce, 0x20, 0xbf, 0x35, 0x37,
+	0x03, 0x86, 0xb7, 0x16, 0x3a, 0x02, 0xfe, 0x16, 0x4e, 0x5a, 0x24, 0x53, 0x1b, 0x32, 0x82, 0x9d,
+	0xb3, 0xf2, 0xf4, 0xe2, 0xa9, 0x8a, 0x76, 0xea, 0x6b, 0xe2, 0x3f, 0x77, 0xe4, 0x76, 0xfa, 0x9f,
+	0x6c, 0xfd, 0x11, 0x96, 0x93, 0x16, 0x7f, 0x02, 0xec, 0x07, 0xee, 0xd2, 0x72, 0x63, 0xf9, 0xf0,
+	0x5b, 0x1f, 0xe6, 0xef, 0xb3, 0x8b, 0x5f, 0x90, 0x7f, 0x1a, 0xc3, 0xc4, 0x5f, 0x41, 0x11, 0xb3,
+	0xc1, 0x57, 0x6a, 0x92, 0xa7, 0xf5, 0x63, 0x35, 0x0d, 0x8d, 0x9c, 0xf1, 0x12, 0xf2, 0xf0, 0xe7,
+	0x7c, 0xa9, 0xee, 0xc7, 0x64, 0xbd, 0x52, 0x93, 0x28, 0xc8, 0x19, 0x7f, 0x01, 0xec, 0xd2, 0x36,
+	0xfc, 0x54, 0xdd, 0x7d, 0xc0, 0xfa, 0x38, 0x6d, 0x24, 0x67, 0x6f, 0xb2, 0xab, 0x22, 0xa4, 0xf8,
+	0xdd, 0xdf, 0x00, 0x00, 0x00, 0xff, 0xff, 0xea, 0xdf, 0xa5, 0x1d, 0xf7, 0x02, 0x00, 0x00,
+}

debug/service/proto/debug.pb.micro.go

@@ -1,5 +1,5 @@
 // Code generated by protoc-gen-micro. DO NOT EDIT.
-// source: micro/go-micro/debug/proto/debug.proto
+// source: micro/go-micro/debug/service/proto/debug.proto
 
 package debug
 
@@ -36,6 +36,7 @@ var _ server.Option
 type DebugService interface {
 	Health(ctx context.Context, in *HealthRequest, opts ...client.CallOption) (*HealthResponse, error)
 	Stats(ctx context.Context, in *StatsRequest, opts ...client.CallOption) (*StatsResponse, error)
+	Log(ctx context.Context, in *LogRequest, opts ...client.CallOption) (Debug_LogService, error)
 }
 
 type debugService struct {
@@ -76,17 +77,63 @@ func (c *debugService) Stats(ctx context.Context, in *StatsRequest, opts ...clie
 	return out, nil
 }
 
+func (c *debugService) Log(ctx context.Context, in *LogRequest, opts ...client.CallOption) (Debug_LogService, error) {
+	req := c.c.NewRequest(c.name, "Debug.Log", &LogRequest{})
+	stream, err := c.c.Stream(ctx, req, opts...)
+	if err != nil {
+		return nil, err
+	}
+	if err := stream.Send(in); err != nil {
+		return nil, err
+	}
+	return &debugServiceLog{stream}, nil
+}
+
+type Debug_LogService interface {
+	SendMsg(interface{}) error
+	RecvMsg(interface{}) error
+	Close() error
+	Recv() (*Record, error)
+}
+
+type debugServiceLog struct {
+	stream client.Stream
+}
+
+func (x *debugServiceLog) Close() error {
+	return x.stream.Close()
+}
+
+func (x *debugServiceLog) SendMsg(m interface{}) error {
+	return x.stream.Send(m)
+}
+
+func (x *debugServiceLog) RecvMsg(m interface{}) error {
+	return x.stream.Recv(m)
+}
+
+func (x *debugServiceLog) Recv() (*Record, error) {
+	m := new(Record)
+	err := x.stream.Recv(m)
+	if err != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
 // Server API for Debug service
 
 type DebugHandler interface {
 	Health(context.Context, *HealthRequest, *HealthResponse) error
 	Stats(context.Context, *StatsRequest, *StatsResponse) error
+	Log(context.Context, *LogRequest, Debug_LogStream) error
 }
 
 func RegisterDebugHandler(s server.Server, hdlr DebugHandler, opts ...server.HandlerOption) error {
 	type debug interface {
 		Health(ctx context.Context, in *HealthRequest, out *HealthResponse) error
 		Stats(ctx context.Context, in *StatsRequest, out *StatsResponse) error
+		Log(ctx context.Context, stream server.Stream) error
 	}
 	type Debug struct {
 		debug
@@ -106,3 +153,38 @@ func (h *debugHandler) Health(ctx context.Context, in *HealthRequest, out *Healt
 func (h *debugHandler) Stats(ctx context.Context, in *StatsRequest, out *StatsResponse) error {
 	return h.DebugHandler.Stats(ctx, in, out)
 }
+
+func (h *debugHandler) Log(ctx context.Context, stream server.Stream) error {
+	m := new(LogRequest)
+	if err := stream.Recv(m); err != nil {
+		return err
+	}
+	return h.DebugHandler.Log(ctx, m, &debugLogStream{stream})
+}
+
+type Debug_LogStream interface {
+	SendMsg(interface{}) error
+	RecvMsg(interface{}) error
+	Close() error
+	Send(*Record) error
+}
+
+type debugLogStream struct {
+	stream server.Stream
+}
+
+func (x *debugLogStream) Close() error {
+	return x.stream.Close()
+}
+
+func (x *debugLogStream) SendMsg(m interface{}) error {
+	return x.stream.Send(m)
+}
+
+func (x *debugLogStream) RecvMsg(m interface{}) error {
+	return x.stream.Recv(m)
+}
+
+func (x *debugLogStream) Send(m *Record) error {
+	return x.stream.Send(m)
+}

debug/service/proto/debug.proto

@@ -0,0 +1,61 @@
+syntax = "proto3";
+
+service Debug {
+        rpc Health(HealthRequest) returns (HealthResponse) {};
+        rpc Stats(StatsRequest) returns (StatsResponse) {};
+        rpc Log(LogRequest) returns (stream Record) {};
+}
+
+message HealthRequest {
+	// optional service name
+	string service = 1;
+}
+
+message HealthResponse {
+	// default: ok
+	string status = 1;
+}
+
+message StatsRequest {
+	// optional service name
+	string service = 1;
+}
+
+message StatsResponse {
+	// timestamp of recording
+	uint64 timestamp = 1;
+	// unix timestamp
+	uint64 started = 2;
+	// in seconds
+	uint64 uptime = 3;
+	// in bytes
+	uint64 memory = 4;
+	// num threads
+	uint64 threads = 5;
+	// total gc in nanoseconds
+	uint64 gc = 6;
+}
+
+// LogRequest requests service logs
+message LogRequest {
+	// service to request logs for
+	string service = 1;
+	// stream records continuously
+	bool stream = 2;
+	// count of records to request
+	int64 count = 3;
+	// relative time in seconds
+	// before the current time
+	// from which to show logs
+	int64 since = 4;
+}
+
+// Record is service log record
+message Record {
+        // timestamp of log record
+        int64 timestamp = 1;
+        // record value
+        string value = 2;
+        // record metadata
+        map<string,string> metadata = 3;
+}