Recovery should be < 500

* Augment the router penalty and decay as a hack fix

* increase recovery cost

api/handler/api/util_test.go

@@ -8,7 +8,7 @@ import (
 
 func TestRequestToProto(t *testing.T) {
 	testData := []*http.Request{
-		&http.Request{
+		{
 			Method: "GET",
 			Header: http.Header{
 				"Header": []string{"test"},

api/handler/http/http_test.go

@@ -27,7 +27,7 @@ func testHttp(t *testing.T, path, service, ns string) {
 	s := &registry.Service{
 		Name: service,
 		Nodes: []*registry.Node{
-			&registry.Node{
+			{
 				Id:      service + "-1",
 				Address: l.Addr().String(),
 			},

api/server/acme/certmagic/certmagic.go

@@ -13,7 +13,7 @@ import (
 )
 
 type certmagicProvider struct {
-	opts *acme.Options
+	opts acme.Options
 }
 
 func (c *certmagicProvider) NewListener(ACMEHosts ...string) (net.Listener, error) {
@@ -40,23 +40,19 @@ func (c *certmagicProvider) NewListener(ACMEHosts ...string) (net.Listener, erro
 
 // New returns a certmagic provider
 func New(options ...acme.Option) acme.Provider {
-	o := &acme.Options{}
-	if len(options) == 0 {
-		for _, op := range acme.Default() {
-			op(o)
-		}
-	} else {
-		for _, op := range options {
-			op(o)
-		}
+	opts := acme.DefaultOptions()
+
+	for _, o := range options {
+		o(&opts)
 	}
-	if o.Cache != nil {
-		if _, ok := o.Cache.(certmagic.Storage); !ok {
+
+	if opts.Cache != nil {
+		if _, ok := opts.Cache.(certmagic.Storage); !ok {
 			log.Fatal("ACME: cache provided doesn't implement certmagic's Storage interface")
 		}
 	}
 
 	return &certmagicProvider{
-		opts: o,
+		opts: opts,
 	}
 }

api/server/acme/certmagic/certmagic_test.go

@@ -11,8 +11,7 @@ import (
 	"github.com/go-acme/lego/v3/providers/dns/cloudflare"
 	"github.com/mholt/certmagic"
 	"github.com/micro/go-micro/api/server/acme"
-	"github.com/micro/go-micro/config/options"
-	cloudflarestorage "github.com/micro/go-micro/store/cloudflare"
+	cfstore "github.com/micro/go-micro/store/cloudflare"
 	"github.com/micro/go-micro/sync/lock/memory"
 )
 
@@ -22,7 +21,7 @@ func TestCertMagic(t *testing.T) {
 	}
 	l, err := New().NewListener()
 	if err != nil {
-		t.Error(err.Error())
+		t.Fatal(err.Error())
 	}
 	l.Close()
 
@@ -34,7 +33,7 @@ func TestCertMagic(t *testing.T) {
 
 	p, err := cloudflare.NewDNSProviderConfig(c)
 	if err != nil {
-		t.Error(err.Error())
+		t.Fatal(err.Error())
 	}
 
 	l, err = New(acme.AcceptToS(true),
@@ -43,7 +42,7 @@ func TestCertMagic(t *testing.T) {
 	).NewListener()
 
 	if err != nil {
-		t.Error(err.Error())
+		t.Fatal(err.Error())
 	}
 	l.Close()
 }
@@ -56,14 +55,11 @@ func TestStorageImplementation(t *testing.T) {
 	}
 
 	var s certmagic.Storage
-	st, err := cloudflarestorage.New(
-		options.WithValue("CF_API_TOKEN", apiToken),
-		options.WithValue("CF_ACCOUNT_ID", accountID),
-		options.WithValue("KV_NAMESPACE_ID", kvID),
+	st := cfstore.NewStore(
+		cfstore.Token(apiToken),
+		cfstore.Account(accountID),
+		cfstore.Namespace(kvID),
 	)
-	if err != nil {
-		t.Fatalf("Couldn't initialise cloudflare storage: %s\n", err.Error())
-	}
 	s = &storage{
 		lock:  memory.NewLock(),
 		store: st,
@@ -71,12 +67,12 @@ func TestStorageImplementation(t *testing.T) {
 
 	// Test Lock
 	if err := s.Lock("test"); err != nil {
-		t.Error(err)
+		t.Fatal(err)
 	}
 
 	// Test Unlock
 	if err := s.Unlock("test"); err != nil {
-		t.Error(err)
+		t.Fatal(err)
 	}
 
 	// Test data
@@ -107,17 +103,17 @@ func TestStorageImplementation(t *testing.T) {
 	// Test Store
 	for _, d := range testdata {
 		if err := s.Store(d.key, d.value); err != nil {
-			t.Error(err.Error())
+			t.Fatal(err.Error())
 		}
 	}
 
 	// Test Load
 	for _, d := range testdata {
 		if value, err := s.Load(d.key); err != nil {
-			t.Error(err.Error())
+			t.Fatal(err.Error())
 		} else {
 			if !reflect.DeepEqual(value, d.value) {
-				t.Errorf("Load %s: expected %v, got %v", d.key, d.value, value)
+				t.Fatalf("Load %s: expected %v, got %v", d.key, d.value, value)
 			}
 		}
 	}
@@ -125,13 +121,13 @@ func TestStorageImplementation(t *testing.T) {
 	// Test Exists
 	for _, d := range testdata {
 		if !s.Exists(d.key) {
-			t.Errorf("%s should exist, but doesn't\n", d.key)
+			t.Fatalf("%s should exist, but doesn't\n", d.key)
 		}
 	}
 
 	// Test List
 	if list, err := s.List("/", true); err != nil {
-		t.Error(err.Error())
+		t.Fatal(err.Error())
 	} else {
 		var expected []string
 		for i, d := range testdata {
@@ -143,16 +139,16 @@ func TestStorageImplementation(t *testing.T) {
 		sort.Strings(expected)
 		sort.Strings(list)
 		if !reflect.DeepEqual(expected, list) {
-			t.Errorf("List: Expected %v, got %v\n", expected, list)
+			t.Fatalf("List: Expected %v, got %v\n", expected, list)
 		}
 	}
 	if list, err := s.List("/foo", false); err != nil {
-		t.Error(err.Error())
+		t.Fatal(err.Error())
 	} else {
 		sort.Strings(list)
 		expected := []string{"/foo/a", "/foo/b", "/foo/bar", "/foo/c", "/foo/d"}
 		if !reflect.DeepEqual(expected, list) {
-			t.Errorf("List: expected %s, got %s\n", expected, list)
+			t.Fatalf("List: expected %s, got %s\n", expected, list)
 		}
 	}
 
@@ -160,16 +156,16 @@ func TestStorageImplementation(t *testing.T) {
 	for _, d := range testdata {
 		info, err := s.Stat(d.key)
 		if err != nil {
-			t.Error(err.Error())
+			t.Fatal(err.Error())
 		} else {
 			if info.Key != d.key {
-				t.Errorf("Stat().Key: expected %s, got %s\n", d.key, info.Key)
+				t.Fatalf("Stat().Key: expected %s, got %s\n", d.key, info.Key)
 			}
 			if info.Size != int64(len(d.value)) {
-				t.Errorf("Stat().Size: expected %d, got %d\n", len(d.value), info.Size)
+				t.Fatalf("Stat().Size: expected %d, got %d\n", len(d.value), info.Size)
 			}
 			if time.Since(info.Modified) > time.Minute {
-				t.Errorf("Stat().Modified: expected time since last modified to be < 1 minute, got %v\n", time.Since(info.Modified))
+				t.Fatalf("Stat().Modified: expected time since last modified to be < 1 minute, got %v\n", time.Since(info.Modified))
 			}
 		}
 
@@ -178,7 +174,7 @@ func TestStorageImplementation(t *testing.T) {
 	// Test Delete
 	for _, d := range testdata {
 		if err := s.Delete(d.key); err != nil {
-			t.Error(err.Error())
+			t.Fatal(err.Error())
 		}
 	}
 
@@ -196,14 +192,11 @@ func TestE2e(t *testing.T) {
 	}
 
 	testLock := memory.NewLock()
-	testStore, err := cloudflarestorage.New(
-		options.WithValue("CF_API_TOKEN", apiToken),
-		options.WithValue("CF_ACCOUNT_ID", accountID),
-		options.WithValue("KV_NAMESPACE_ID", kvID),
+	testStore := cfstore.NewStore(
+		cfstore.Token(apiToken),
+		cfstore.Account(accountID),
+		cfstore.Namespace(kvID),
 	)
-	if err != nil {
-		t.Fatal(err.Error())
-	}
 	testStorage := NewStorage(testLock, testStore)
 
 	conf := cloudflare.NewDefaultConfig()

api/server/acme/certmagic/storage.go

@@ -89,7 +89,7 @@ func (s *storage) Exists(key string) bool {
 }
 
 func (s *storage) List(prefix string, recursive bool) ([]string, error) {
-	records, err := s.store.Sync()
+	records, err := s.store.List()
 	if err != nil {
 		return nil, err
 	}

api/server/acme/options.go

@@ -63,11 +63,11 @@ func Cache(c interface{}) Option {
 	}
 }
 
-// Default uses the Let's Encrypt Production CA, with DNS Challenge disabled.
-func Default() []Option {
-	return []Option{
-		AcceptToS(true),
-		CA(LetsEncryptProductionCA),
-		OnDemand(true),
+// DefaultOptions uses the Let's Encrypt Production CA, with DNS Challenge disabled.
+func DefaultOptions() Options {
+	return Options{
+		AcceptToS: true,
+		CA:        LetsEncryptProductionCA,
+		OnDemand:  true,
 	}
 }

broker/common_test.go

@@ -7,7 +7,7 @@ import (
 var (
 	// mock data
 	testData = map[string][]*registry.Service{
-		"foo": []*registry.Service{
+		"foo": {
 			{
 				Name:    "foo",
 				Version: "1.0.0",

broker/http_broker_test.go

@@ -125,7 +125,7 @@ func pub(be *testing.B, c int) {
 
 	for i := 0; i < c; i++ {
 		go func() {
-			for _ = range ch {
+			for range ch {
 				if err := b.Publish(topic, msg); err != nil {
 					be.Fatalf("Unexpected publish error: %v", err)
 				}

client/common_test.go

@@ -7,7 +7,7 @@ import (
 var (
 	// mock data
 	testData = map[string][]*registry.Service{
-		"foo": []*registry.Service{
+		"foo": {
 			{
 				Name:    "foo",
 				Version: "1.0.0",

client/grpc/grpc_test.go

@@ -42,7 +42,7 @@ func TestGRPCClient(t *testing.T) {
 		Name:    "helloworld",
 		Version: "test",
 		Nodes: []*registry.Node{
-			&registry.Node{
+			{
 				Id:      "test-1",
 				Address: l.Addr().String(),
 			},

client/rpc_client_test.go

@@ -143,7 +143,7 @@ func TestCallWrapper(t *testing.T) {
 		Name:    service,
 		Version: "latest",
 		Nodes: []*registry.Node{
-			&registry.Node{
+			{
 				Id:      id,
 				Address: address,
 			},

client/selector/common_test.go

@@ -7,7 +7,7 @@ import (
 var (
 	// mock data
 	testData = map[string][]*registry.Service{
-		"foo": []*registry.Service{
+		"foo": {
 			{
 				Name:    "foo",
 				Version: "1.0.0",

client/selector/dns/dns.go

@@ -72,7 +72,7 @@ func (d *dnsSelector) Select(service string, opts ...selector.SelectOption) (sel
 	}
 
 	services := []*registry.Service{
-		&registry.Service{
+		{
 			Name:  service,
 			Nodes: nodes,
 		},

client/selector/filter_test.go

@@ -14,20 +14,20 @@ func TestFilterEndpoint(t *testing.T) {
 	}{
 		{
 			services: []*registry.Service{
-				&registry.Service{
+				{
 					Name:    "test",
 					Version: "1.0.0",
 					Endpoints: []*registry.Endpoint{
-						&registry.Endpoint{
+						{
 							Name: "Foo.Bar",
 						},
 					},
 				},
-				&registry.Service{
+				{
 					Name:    "test",
 					Version: "1.1.0",
 					Endpoints: []*registry.Endpoint{
-						&registry.Endpoint{
+						{
 							Name: "Baz.Bar",
 						},
 					},
@@ -38,20 +38,20 @@ func TestFilterEndpoint(t *testing.T) {
 		},
 		{
 			services: []*registry.Service{
-				&registry.Service{
+				{
 					Name:    "test",
 					Version: "1.0.0",
 					Endpoints: []*registry.Endpoint{
-						&registry.Endpoint{
+						{
 							Name: "Foo.Bar",
 						},
 					},
 				},
-				&registry.Service{
+				{
 					Name:    "test",
 					Version: "1.1.0",
 					Endpoints: []*registry.Endpoint{
-						&registry.Endpoint{
+						{
 							Name: "Foo.Bar",
 						},
 					},
@@ -95,11 +95,11 @@ func TestFilterLabel(t *testing.T) {
 	}{
 		{
 			services: []*registry.Service{
-				&registry.Service{
+				{
 					Name:    "test",
 					Version: "1.0.0",
 					Nodes: []*registry.Node{
-						&registry.Node{
+						{
 							Id:      "test-1",
 							Address: "localhost",
 							Metadata: map[string]string{
@@ -108,11 +108,11 @@ func TestFilterLabel(t *testing.T) {
 						},
 					},
 				},
-				&registry.Service{
+				{
 					Name:    "test",
 					Version: "1.1.0",
 					Nodes: []*registry.Node{
-						&registry.Node{
+						{
 							Id:      "test-2",
 							Address: "localhost",
 							Metadata: map[string]string{
@@ -127,21 +127,21 @@ func TestFilterLabel(t *testing.T) {
 		},
 		{
 			services: []*registry.Service{
-				&registry.Service{
+				{
 					Name:    "test",
 					Version: "1.0.0",
 					Nodes: []*registry.Node{
-						&registry.Node{
+						{
 							Id:      "test-1",
 							Address: "localhost",
 						},
 					},
 				},
-				&registry.Service{
+				{
 					Name:    "test",
 					Version: "1.1.0",
 					Nodes: []*registry.Node{
-						&registry.Node{
+						{
 							Id:      "test-2",
 							Address: "localhost",
 						},
@@ -187,11 +187,11 @@ func TestFilterVersion(t *testing.T) {
 	}{
 		{
 			services: []*registry.Service{
-				&registry.Service{
+				{
 					Name:    "test",
 					Version: "1.0.0",
 				},
-				&registry.Service{
+				{
 					Name:    "test",
 					Version: "1.1.0",
 				},
@@ -201,11 +201,11 @@ func TestFilterVersion(t *testing.T) {
 		},
 		{
 			services: []*registry.Service{
-				&registry.Service{
+				{
 					Name:    "test",
 					Version: "1.0.0",
 				},
-				&registry.Service{
+				{
 					Name:    "test",
 					Version: "1.1.0",
 				},

client/selector/router/router.go

@@ -111,7 +111,7 @@ func (r *routerSelector) getRoutes(service string) ([]router.Route, error) {
 			Gateway: r.Gateway,
 			Network: r.Network,
 			Link:    r.Link,
-			Metric:  int(r.Metric),
+			Metric:  r.Metric,
 		})
 	}
 

client/selector/strategy_test.go

@@ -8,29 +8,29 @@ import (
 
 func TestStrategies(t *testing.T) {
 	testData := []*registry.Service{
-		&registry.Service{
+		{
 			Name:    "test1",
 			Version: "latest",
 			Nodes: []*registry.Node{
-				&registry.Node{
+				{
 					Id:      "test1-1",
 					Address: "10.0.0.1:1001",
 				},
-				&registry.Node{
+				{
 					Id:      "test1-2",
 					Address: "10.0.0.2:1002",
 				},
 			},
 		},
-		&registry.Service{
+		{
 			Name:    "test1",
 			Version: "default",
 			Nodes: []*registry.Node{
-				&registry.Node{
+				{
 					Id:      "test1-3",
 					Address: "10.0.0.3:1003",
 				},
-				&registry.Node{
+				{
 					Id:      "test1-4",
 					Address: "10.0.0.4:1004",
 				},

common_test.go

@@ -7,7 +7,7 @@ import (
 var (
 	// mock data
 	testData = map[string][]*registry.Service{
-		"foo": []*registry.Service{
+		"foo": {
 			{
 				Name:    "foo",
 				Version: "1.0.0",

config/cmd/cmd.go

@@ -27,7 +27,6 @@ import (
 
 	// registries
 	"github.com/micro/go-micro/registry"
-	"github.com/micro/go-micro/registry/consul"
 	"github.com/micro/go-micro/registry/etcd"
 	"github.com/micro/go-micro/registry/mdns"
 	rmem "github.com/micro/go-micro/registry/memory"
@@ -45,6 +44,10 @@ import (
 	thttp "github.com/micro/go-micro/transport/http"
 	tmem "github.com/micro/go-micro/transport/memory"
 	"github.com/micro/go-micro/transport/quic"
+
+	// runtimes
+	"github.com/micro/go-micro/runtime"
+	"github.com/micro/go-micro/runtime/kubernetes"
 )
 
 type Cmd interface {
@@ -68,6 +71,12 @@ var (
 	DefaultCmd = newCmd()
 
 	DefaultFlags = []cli.Flag{
+		cli.StringFlag{
+			Name:   "runtime",
+			Usage:  "Micro runtime",
+			EnvVar: "MICRO_RUNTIME",
+			Value:  "local",
+		},
 		cli.StringFlag{
 			Name:   "client",
 			EnvVar: "MICRO_CLIENT",
@@ -155,7 +164,7 @@ var (
 		cli.StringFlag{
 			Name:   "registry",
 			EnvVar: "MICRO_REGISTRY",
-			Usage:  "Registry for discovery. consul, etcd, mdns",
+			Usage:  "Registry for discovery. etcd, mdns",
 		},
 		cli.StringFlag{
 			Name:   "registry_address",
@@ -196,7 +205,6 @@ var (
 	DefaultRegistries = map[string]func(...registry.Option) registry.Registry{
 		"go.micro.registry": regSrv.NewRegistry,
 		"service":           regSrv.NewRegistry,
-		"consul":            consul.NewRegistry,
 		"etcd":              etcd.NewRegistry,
 		"mdns":              mdns.NewRegistry,
 		"memory":            rmem.NewRegistry,
@@ -223,6 +231,11 @@ var (
 		"quic":   quic.NewTransport,
 	}
 
+	DefaultRuntimes = map[string]func(...runtime.Option) runtime.Runtime{
+		"local":      runtime.NewRuntime,
+		"kubernetes": kubernetes.NewRuntime,
+	}
+
 	// used for default selection as the fall back
 	defaultClient    = "rpc"
 	defaultServer    = "rpc"
@@ -230,6 +243,7 @@ var (
 	defaultRegistry  = "mdns"
 	defaultSelector  = "registry"
 	defaultTransport = "http"
+	defaultRuntime   = "local"
 )
 
 func init() {
@@ -249,6 +263,7 @@ func newCmd(opts ...Option) Cmd {
 		Server:    &server.DefaultServer,
 		Selector:  &selector.DefaultSelector,
 		Transport: &transport.DefaultTransport,
+		Runtime:   &runtime.DefaultRuntime,
 
 		Brokers:    DefaultBrokers,
 		Clients:    DefaultClients,
@@ -256,6 +271,7 @@ func newCmd(opts ...Option) Cmd {
 		Selectors:  DefaultSelectors,
 		Servers:    DefaultServers,
 		Transports: DefaultTransports,
+		Runtimes:   DefaultRuntimes,
 	}
 
 	for _, o := range opts {
@@ -296,6 +312,16 @@ func (c *cmd) Before(ctx *cli.Context) error {
 	var serverOpts []server.Option
 	var clientOpts []client.Option
 
+	// Set the runtime
+	if name := ctx.String("runtime"); len(name) > 0 {
+		r, ok := c.opts.Runtimes[name]
+		if !ok {
+			return fmt.Errorf("Unsupported runtime: %s", name)
+		}
+
+		*c.opts.Runtime = r()
+	}
+
 	// Set the client
 	if name := ctx.String("client"); len(name) > 0 {
 		// only change if we have the client and type differs

config/cmd/options.go

@@ -7,6 +7,7 @@ import (
 	"github.com/micro/go-micro/client"
 	"github.com/micro/go-micro/client/selector"
 	"github.com/micro/go-micro/registry"
+	"github.com/micro/go-micro/runtime"
 	"github.com/micro/go-micro/server"
 	"github.com/micro/go-micro/transport"
 )
@@ -24,6 +25,7 @@ type Options struct {
 	Transport *transport.Transport
 	Client    *client.Client
 	Server    *server.Server
+	Runtime   *runtime.Runtime
 
 	Brokers    map[string]func(...broker.Option) broker.Broker
 	Clients    map[string]func(...client.Option) client.Client
@@ -31,6 +33,7 @@ type Options struct {
 	Selectors  map[string]func(...selector.Option) selector.Selector
 	Servers    map[string]func(...server.Option) server.Server
 	Transports map[string]func(...transport.Option) transport.Transport
+	Runtimes   map[string]func(...runtime.Option) runtime.Runtime
 
 	// Other options for implementations of the interface
 	// can be stored in a context
@@ -135,3 +138,10 @@ func NewTransport(name string, t func(...transport.Option) transport.Transport)
 		o.Transports[name] = t
 	}
 }
+
+// New runtime func
+func NewRuntime(name string, r func(...runtime.Option) runtime.Runtime) Option {
+	return func(o *Options) {
+		o.Runtimes[name] = r
+	}
+}

config/reader/json/values_test.go

@@ -62,7 +62,7 @@ func TestStructArray(t *testing.T) {
 		{
 			[]byte(`[{"foo": "bar"}]`),
 			emptyTSlice,
-			[]T{T{Foo: "bar"}},
+			[]T{{Foo: "bar"}},
 		},
 	}
 

errors/errors_test.go

@@ -7,7 +7,7 @@ import (
 
 func TestErrors(t *testing.T) {
 	testData := []*Error{
-		&Error{
+		{
 			Id:     "test",
 			Code:   500,
 			Detail: "Internal server error",

go.mod

@@ -8,7 +8,6 @@ require (
 	github.com/bitly/go-simplejson v0.5.0
 	github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 // indirect
 	github.com/bwmarrin/discordgo v0.19.0
-	github.com/cloudflare/cloudflare-go v0.10.4
 	github.com/coreos/bbolt v1.3.3 // indirect
 	github.com/coreos/etcd v3.3.17+incompatible
 	github.com/coreos/go-semver v0.3.0 // indirect
@@ -30,13 +29,14 @@ require (
 	github.com/gorilla/websocket v1.4.1
 	github.com/grpc-ecosystem/go-grpc-middleware v1.1.0 // indirect
 	github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect
-	github.com/hashicorp/consul/api v1.2.0
 	github.com/hashicorp/hcl v1.0.0
 	github.com/imdario/mergo v0.3.8
 	github.com/jonboulle/clockwork v0.1.0 // indirect
 	github.com/joncalhoun/qson v0.0.0-20170526102502-8a9cab3a62b1
 	github.com/json-iterator/go v1.1.7
+	github.com/kr/pretty v0.1.0
 	github.com/leodido/go-urn v1.1.0 // indirect
+	github.com/lib/pq v1.2.0
 	github.com/lucas-clemente/quic-go v0.12.1
 	github.com/mholt/certmagic v0.7.5
 	github.com/micro/cli v0.2.0

go.sum

@@ -36,10 +36,6 @@ github.com/aliyun/aliyun-oss-go-sdk v0.0.0-20190307165228-86c17b95fcd5/go.mod h1
 github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239 h1:kFOfPq6dUM1hTo4JG6LR5AXSUEsOjtdm0kw0FtQtMJA=
 github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c=
 github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ=
-github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o=
-github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da h1:8GUt8eRujhVEGZFFEjBj46YV4rDjvGrNxb0KMWYkL2I=
-github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY=
-github.com/armon/go-radix v0.0.0-20180808171621-7fddfc383310/go.mod h1:ufUuZ+zHj4x4TnLV4JWEpy2hxWSpsRywHrMgIH9cCH8=
 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPdPJAN/hZIm0C4OItdklCFmMRWYpio=
 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs=
 github.com/aws/aws-sdk-go v1.23.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo=
@@ -50,7 +46,6 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24
 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8=
 github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM=
 github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw=
-github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs=
 github.com/bitly/go-simplejson v0.5.0 h1:6IH+V8/tVMab511d5bn4M7EwGXZf9Hj6i2xSwkNEM+Y=
 github.com/bitly/go-simplejson v0.5.0/go.mod h1:cXHtHw4XUPsvGaxgjIAn8PhEWG9NfngEKAMDJEczWVA=
 github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY=
@@ -65,8 +60,6 @@ github.com/cheekybits/genny v1.0.0/go.mod h1:+tQajlRqAUrPI7DOSpB0XAqZYtQakVtB7wX
 github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw=
 github.com/cloudflare/cloudflare-go v0.10.2 h1:VBodKICVPnwmDxstcW3biKcDSpFIfS/RELUXsZSBYK4=
 github.com/cloudflare/cloudflare-go v0.10.2/go.mod h1:qhVI5MKwBGhdNU89ZRz2plgYutcJ5PCekLxXn56w6SY=
-github.com/cloudflare/cloudflare-go v0.10.4 h1:7C1D9mtcNFZLCqmhkHK2BlwKKm9fi4cBqY6qpYtQv5E=
-github.com/cloudflare/cloudflare-go v0.10.4/go.mod h1:4HgmUutVbZTRnHg91bS8lvlA0Wx+TgqttLDcwey2S6E=
 github.com/containerd/continuity v0.0.0-20181203112020-004b46473808 h1:4BX8f882bXEDKfWIf0wa8HRvpnBoPszJJXL+TVbBw4M=
 github.com/containerd/continuity v0.0.0-20181203112020-004b46473808/go.mod h1:GL3xCUCBDV3CZiTSEKksMWbLE66hEyuu9qyDOOqM47Y=
 github.com/coreos/bbolt v1.3.3 h1:n6AiVyVRKQFNb6mJlwESEvvLoDyiTzXX7ORAUlkeBdY=
@@ -105,7 +98,6 @@ github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFP
 github.com/emirpasic/gods v1.12.0 h1:QAUIPSaCu4G+POclxeqb3F+WPpdKqFGlw36+yOzGlrg=
 github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o=
 github.com/exoscale/egoscale v0.18.1/go.mod h1:Z7OOdzzTOz1Q1PjQXumlz9Wn/CddH0zSYdCF3rnBKXE=
-github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4=
 github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M=
 github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 h1:BHsljHzVlRcyQhjrss6TZTdY2VfCqZPbv5k3iBFa2ZQ=
 github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc=
@@ -187,41 +179,12 @@ github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgf
 github.com/grpc-ecosystem/grpc-gateway v1.8.5 h1:2+KSC78XiO6Qy0hIjfc1OD9H+hsaJdJlb8Kqsd41CTE=
 github.com/grpc-ecosystem/grpc-gateway v1.8.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
 github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542/go.mod h1:Ow0tF8D4Kplbc8s8sSb3V2oUCygFHVp8gC3Dn6U4MNI=
-github.com/hashicorp/consul/api v1.2.0 h1:oPsuzLp2uk7I7rojPKuncWbZ+m5TMoD4Ivs+2Rkeh4Y=
-github.com/hashicorp/consul/api v1.2.0/go.mod h1:1SIkFYi2ZTXUE5Kgt179+4hH33djo11+0Eo2XgTAtkw=
-github.com/hashicorp/consul/sdk v0.2.0 h1:GWFYFmry/k4b1hEoy7kSkmU8e30GAyI4VZHk0fRxeL4=
-github.com/hashicorp/consul/sdk v0.2.0/go.mod h1:VKf9jXwCTEY1QZP2MOLRhb5i/I/ssyNV1vwHyQBF0x8=
-github.com/hashicorp/errwrap v1.0.0 h1:hLrqtEDnRye3+sgx6z4qVLNuviH3MR5aQ0ykNJa/UYA=
-github.com/hashicorp/errwrap v1.0.0/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4=
-github.com/hashicorp/go-cleanhttp v0.5.1 h1:dH3aiDG9Jvb5r5+bYHsikaOUIpcM0xvgMXVoDkXMzJM=
-github.com/hashicorp/go-cleanhttp v0.5.1/go.mod h1:JpRdi6/HCYpAwUzNwuwqhbovhLtngrth3wmdIIUrZ80=
-github.com/hashicorp/go-immutable-radix v1.0.0 h1:AKDB1HM5PWEA7i4nhcpwOrO2byshxBjXVn/J/3+z5/0=
-github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60=
-github.com/hashicorp/go-msgpack v0.5.3 h1:zKjpN5BK/P5lMYrLmBHdBULWbJ0XpYR+7NGzqkZzoD4=
-github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iPBM1vqhUKIvfM=
-github.com/hashicorp/go-multierror v1.0.0 h1:iVjPR7a6H0tWELX5NxNe7bYopibicUzc7uPribsnS6o=
-github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk=
-github.com/hashicorp/go-rootcerts v1.0.0 h1:Rqb66Oo1X/eSV1x66xbDccZjhJigjg0+e82kpwzSwCI=
-github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU=
-github.com/hashicorp/go-sockaddr v1.0.0 h1:GeH6tui99pF4NJgfnhp+L6+FfobzVW3Ah46sLo0ICXs=
-github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU=
-github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4=
-github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
-github.com/hashicorp/go-uuid v1.0.1 h1:fv1ep09latC32wFoVwnqcnKJGnMSdBanPczbHAYm1BE=
-github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro=
-github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=
 github.com/hashicorp/golang-lru v0.5.3 h1:YPkqC67at8FYaadspW/6uE0COsBxS2656RLEr8Bppgk=
 github.com/hashicorp/golang-lru v0.5.3/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4=
 github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4=
 github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ=
-github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64=
-github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0mNTz8vQ=
-github.com/hashicorp/memberlist v0.1.3 h1:EmmoJme1matNzb+hMpDuR/0sbJSUisxyqBGG676r31M=
-github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I=
-github.com/hashicorp/serf v0.8.2 h1:YZ7UKsJv+hKjqGVUUbtE3HNj79Eln2oQ75tniF6iPt0=
-github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc=
 github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
 github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
 github.com/iij/doapi v0.0.0-20190504054126-0bbf12d6d7df/go.mod h1:QMZY7/J/KSQEhKWFeDesPjMj+wCHReeknARU3wqlyN4=
@@ -265,6 +228,8 @@ github.com/labbsr0x/bindman-dns-webhook v1.0.2/go.mod h1:p6b+VCXIR8NYKpDr8/dg1HK
 github.com/labbsr0x/goh v1.0.1/go.mod h1:8K2UhVoaWXcCU7Lxoa2omWnC8gyW8px7/lmO61c027w=
 github.com/leodido/go-urn v1.1.0 h1:Sm1gr51B1kKyfD2BlRcLSiEkffoG96g6TPv6eRoEiB8=
 github.com/leodido/go-urn v1.1.0/go.mod h1:+cyI34gQWZcE1eQU7NVgKkkzdXDQHr1dBMtdAPozLkw=
+github.com/lib/pq v1.2.0 h1:LXpIM/LZ5xGFhOpXAQUIMM1HdyqzVYM13zNdjCEEcA0=
+github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo=
 github.com/linode/linodego v0.10.0/go.mod h1:cziNP7pbvE3mXIPneHj0oRY8L1WtGEIKlZ8LANE4eXA=
 github.com/liquidweb/liquidweb-go v1.6.0/go.mod h1:UDcVnAMDkZxpw4Y7NOHkqoeiGacVLEIG/i5J9cyixzQ=
 github.com/lucas-clemente/quic-go v0.12.1 h1:BPITli+6KnKogtTxBk2aS4okr5dUHz2LtIDAP1b8UL4=
@@ -272,7 +237,6 @@ github.com/lucas-clemente/quic-go v0.12.1/go.mod h1:UXJJPE4RfFef/xPO5wQm0tITK8gN
 github.com/marten-seemann/qpack v0.1.0/go.mod h1:LFt1NU/Ptjip0C2CPkhimBz5CGE3WGDAUWqna+CNTrI=
 github.com/marten-seemann/qtls v0.3.2 h1:O7awy4bHEzSX/K3h+fZig3/Vo03s/RxlxgsAk9sYamI=
 github.com/marten-seemann/qtls v0.3.2/go.mod h1:xzjG7avBwGGbdZ8dTGxlBnLArsVKLvwmjgmPuiQEcYk=
-github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU=
 github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4=
 github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
 github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU=
@@ -285,22 +249,14 @@ github.com/micro/cli v0.2.0 h1:ut3rV5JWqZjsXIa2MvGF+qMUP8DAUTvHX9Br5gO4afA=
 github.com/micro/cli v0.2.0/go.mod h1:jRT9gmfVKWSS6pkKcXQ8YhUyj6bzwxK8Fp5b0Y7qNnk=
 github.com/micro/mdns v0.3.0 h1:bYycYe+98AXR3s8Nq5qvt6C573uFTDPIYzJemWON0QE=
 github.com/micro/mdns v0.3.0/go.mod h1:KJ0dW7KmicXU2BV++qkLlmHYcVv7/hHnbtguSWt9Aoc=
-github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
 github.com/miekg/dns v1.1.3/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
 github.com/miekg/dns v1.1.15 h1:CSSIDtllwGLMoA6zjdKnaE6Tx6eVUxQ29LUgGetiDCI=
 github.com/miekg/dns v1.1.15/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
-github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc=
-github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
-github.com/mitchellh/go-testing-interface v1.0.0 h1:fzU/JVNcaqHQEcVFAKeR41fkiLdIPrefOvVG1VZ96U0=
-github.com/mitchellh/go-testing-interface v1.0.0/go.mod h1:kRemZodwjscx+RGhAo8eIhFbs2+BFgRtFPeD/KE+zxI=
 github.com/mitchellh/go-vnc v0.0.0-20150629162542-723ed9867aed/go.mod h1:3rdaFaCv4AyBgu5ALFM0+tSuHrBh6v692nyQe3ikrq0=
-github.com/mitchellh/gox v0.4.0/go.mod h1:Sd9lOJ0+aimLBi73mGofS1ycjY8lL3uZM3JPS42BGNg=
 github.com/mitchellh/hashstructure v1.0.0 h1:ZkRJX1CyOoTkar7p/mLS5TZU4nJ1Rn/F8u9dGS02Q3Y=
 github.com/mitchellh/hashstructure v1.0.0/go.mod h1:QjSHrPWS+BGUVBYkbTZWEnOh3G1DutKwClXU/ABz6AQ=
-github.com/mitchellh/iochan v1.0.0/go.mod h1:JwYml1nuB7xOzsp52dPpHFffvOCDupsG0QubkSMEySY=
-github.com/mitchellh/mapstructure v0.0.0-20160808181253-ca63d7c062ee/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
 github.com/mitchellh/mapstructure v1.1.2 h1:fmNYVwqnSfB9mZU6OS2O6GsXM+wcskZDuKQzvN1EDeE=
 github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
 github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
@@ -345,8 +301,6 @@ github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFSt
 github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJc5AZX7/PBEpw=
 github.com/oracle/oci-go-sdk v7.0.0+incompatible/go.mod h1:VQb79nF8Z2cwLkLS35ukwStZIg5F66tcBccjip/j888=
 github.com/ovh/go-ovh v0.0.0-20181109152953-ba5adb4cf014/go.mod h1:joRatxRJaZBsY3JAOEMcoOp05CnZzsx4scTxi95DHyQ=
-github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c h1:Lgl0gzECD8GnQ5QCWA8o6BtfL6mDH5rQgM4/fX3avOs=
-github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
 github.com/pelletier/go-buffruneio v0.2.0/go.mod h1:JkE26KsDizTr40EUHkXVtNPvgGtbSNq5BcowyYOWdKo=
 github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY=
 github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
@@ -354,7 +308,6 @@ github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I=
 github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0=
 github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM=
 github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4=
-github.com/posener/complete v1.1.1/go.mod h1:em0nMJCgc9GFtwrmVmEMR/ZL6WyhyjMBndrE9hABlRI=
 github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw=
 github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs=
 github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
@@ -377,11 +330,8 @@ github.com/rainycape/memcache v0.0.0-20150622160815-1031fa0ce2f2/go.mod h1:7tZKc
 github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
-github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts=
 github.com/sacloud/libsacloud v1.26.1/go.mod h1:79ZwATmHLIFZIMd7sxA3LwzVy/B77uj3LDoToVTxDoQ=
 github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
-github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529 h1:nn5Wsu0esKSJiIVhscUtVbo7ada43DJhG55ua/hjS5I=
-github.com/sean-/seed v0.0.0-20170313163322-e2103e2c3529/go.mod h1:DxrIzT+xaE7yg65j358z/aeFdxmN0P9QXhEzd20vsDc=
 github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ=
 github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo=
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
@@ -436,7 +386,6 @@ go.uber.org/zap v1.10.0 h1:ORx85nbTijNz8ljznvCMR1ZBIPKFn3jQrag10X2AsuM=
 go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q=
 golang.org/x/crypto v0.0.0-20180621125126-a49355c7e3f8/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
-golang.org/x/crypto v0.0.0-20181029021203-45a5f77698d3/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20181030102418-4d3f4d9ffa16/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20181203042331-505ab145d0a9/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
 golang.org/x/crypto v0.0.0-20190130090550-b01c7a725664/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
@@ -459,9 +408,7 @@ golang.org/x/net v0.0.0-20180611182652-db08ff08e862/go.mod h1:mL1N/T3taQHkDXs73r
 golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20181023162649-9b4f9f5ad519/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181114220301-adae6a3d119a/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
-golang.org/x/net v0.0.0-20181201002055-351d144fa1fc/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20181220203305-927f97764cc3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190108225652-1e06a53dbb7e/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
 golang.org/x/net v0.0.0-20190125091013-d26f9f9a57f3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4=
@@ -474,7 +421,6 @@ golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLL
 golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20190930134127-c5a3c61f89f3/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
-golang.org/x/net v0.0.0-20191007182048-72f939374954/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/net v0.0.0-20191011234655-491137f69257 h1:ry8e2D+cwaV6hk7lb3aRTjjZo24shrbK0e11QEOkTIg=
 golang.org/x/net v0.0.0-20191011234655-491137f69257/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
 golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
@@ -487,11 +433,9 @@ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJ
 golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU=
 golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM=
 golang.org/x/sys v0.0.0-20180622082034-63fc586f45fe/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
-golang.org/x/sys v0.0.0-20181026203630-95b1ffbd15a5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=
 golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY=

metadata/metadata.go

@@ -12,6 +12,7 @@ type metaKey struct{}
 // from Transport headers.
 type Metadata map[string]string
 
+// Copy makes a copy of the metadata
 func Copy(md Metadata) Metadata {
 	cmd := make(Metadata)
 	for k, v := range md {
@@ -20,11 +21,31 @@ func Copy(md Metadata) Metadata {
 	return cmd
 }
 
+// FromContext returns metadata from the given context
 func FromContext(ctx context.Context) (Metadata, bool) {
 	md, ok := ctx.Value(metaKey{}).(Metadata)
 	return md, ok
 }
 
+// NewContext creates a new context with the given metadata
 func NewContext(ctx context.Context, md Metadata) context.Context {
 	return context.WithValue(ctx, metaKey{}, md)
 }
+
+// MergeContext merges metadata to existing metadata, overwriting if specified
+func MergeContext(ctx context.Context, patchMd Metadata, overwrite bool) context.Context {
+	md, _ := ctx.Value(metaKey{}).(Metadata)
+	cmd := make(Metadata)
+	for k, v := range md {
+		cmd[k] = v
+	}
+	for k, v := range patchMd {
+		if _, ok := cmd[k]; ok && !overwrite {
+			// skip
+		} else {
+			cmd[k] = v
+		}
+	}
+	return context.WithValue(ctx, metaKey{}, cmd)
+
+}

metadata/metadata_test.go

@@ -2,6 +2,7 @@ package metadata
 
 import (
 	"context"
+	"reflect"
 	"testing"
 )
 
@@ -40,3 +41,42 @@ func TestMetadataContext(t *testing.T) {
 		t.Errorf("Expected metadata length 1 got %d", i)
 	}
 }
+
+func TestMergeContext(t *testing.T) {
+	type args struct {
+		existing  Metadata
+		append    Metadata
+		overwrite bool
+	}
+	tests := []struct {
+		name string
+		args args
+		want Metadata
+	}{
+		{
+			name: "matching key, overwrite false",
+			args: args{
+				existing:  Metadata{"foo": "bar", "sumo": "demo"},
+				append:    Metadata{"sumo": "demo2"},
+				overwrite: false,
+			},
+			want: Metadata{"foo": "bar", "sumo": "demo"},
+		},
+		{
+			name: "matching key, overwrite true",
+			args: args{
+				existing:  Metadata{"foo": "bar", "sumo": "demo"},
+				append:    Metadata{"sumo": "demo2"},
+				overwrite: true,
+			},
+			want: Metadata{"foo": "bar", "sumo": "demo2"},
+		},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			if got, _ := FromContext(MergeContext(NewContext(context.TODO(), tt.args.existing), tt.args.append, tt.args.overwrite)); !reflect.DeepEqual(got, tt.want) {
+				t.Errorf("MergeContext() = %v, want %v", got, tt.want)
+			}
+		})
+	}
+}

monitor/default.go

@@ -78,13 +78,6 @@ func (m *monitor) check(service string) (*Status, error) {
 				client.WithRetries(3),
 			)
 			if err != nil {
-				// reap the dead node
-				m.registry.Deregister(&registry.Service{
-					Name:    service.Name,
-					Version: service.Version,
-					Nodes:   []*registry.Node{node},
-				})
-
 				// save the error
 				gerr = err
 				continue
@@ -140,7 +133,7 @@ func (m *monitor) reap() {
 	defer m.Unlock()
 
 	// range over our watched services
-	for service, _ := range m.services {
+	for service := range m.services {
 		// check if the service exists in the registry
 		if !serviceMap[service] {
 			// if not, delete it in our status map
@@ -195,14 +188,14 @@ func (m *monitor) run() {
 			serviceMap := make(map[string]bool)
 
 			m.RLock()
-			for service, _ := range m.services {
+			for service := range m.services {
 				serviceMap[service] = true
 			}
 			m.RUnlock()
 
 			go func() {
 				// check the status of all watched services
-				for service, _ := range serviceMap {
+				for service := range serviceMap {
 					select {
 					case <-m.exit:
 						return
@@ -307,7 +300,7 @@ func (m *monitor) Stop() error {
 		return nil
 	default:
 		close(m.exit)
-		for s, _ := range m.services {
+		for s := range m.services {
 			delete(m.services, s)
 		}
 		m.registry.Stop()

network/default.go

@@ -4,6 +4,7 @@ import (
 	"errors"
 	"fmt"
 	"hash/fnv"
+	"math"
 	"sync"
 	"time"
 
@@ -37,6 +38,8 @@ var (
 var (
 	// ErrClientNotFound is returned when client for tunnel channel could not be found
 	ErrClientNotFound = errors.New("client not found")
+	// ErrPeerLinkNotFound is returned when peer link could not be found in tunnel Links
+	ErrPeerLinkNotFound = errors.New("peer link not found")
 )
 
 // network implements Network interface
@@ -58,6 +61,8 @@ type network struct {
 
 	// tunClient is a map of tunnel clients keyed over tunnel channel names
 	tunClient map[string]transport.Client
+	// peerLinks is a map of links for each peer
+	peerLinks map[string]tunnel.Link
 
 	sync.RWMutex
 	// connected marks the network as connected
@@ -74,21 +79,6 @@ func newNetwork(opts ...Option) Network {
 		o(&options)
 	}
 
-	// init tunnel address to the network bind address
-	options.Tunnel.Init(
-		tunnel.Address(options.Address),
-	)
-
-	// init router Id to the network id
-	options.Router.Init(
-		router.Id(options.Id),
-	)
-
-	// create tunnel client with tunnel transport
-	tunTransport := tun.NewTransport(
-		tun.WithTunnel(options.Tunnel),
-	)
-
 	// set the address to a hashed address
 	hasher := fnv.New64()
 	hasher.Write([]byte(options.Address + options.Id))
@@ -106,6 +96,22 @@ func newNetwork(opts ...Option) Network {
 		peerAddress = address
 	}
 
+	// init tunnel address to the network bind address
+	options.Tunnel.Init(
+		tunnel.Address(options.Address),
+	)
+
+	// init router Id to the network id
+	options.Router.Init(
+		router.Id(options.Id),
+		router.Address(peerAddress),
+	)
+
+	// create tunnel client with tunnel transport
+	tunTransport := tun.NewTransport(
+		tun.WithTunnel(options.Tunnel),
+	)
+
 	// server is network server
 	server := server.NewServer(
 		server.Id(options.Id),
@@ -138,6 +144,7 @@ func newNetwork(opts ...Option) Network {
 		server:    server,
 		client:    client,
 		tunClient: make(map[string]transport.Client),
+		peerLinks: make(map[string]tunnel.Link),
 	}
 
 	network.node.network = network
@@ -246,19 +253,22 @@ func (n *network) resolve() {
 }
 
 // handleNetConn handles network announcement messages
-func (n *network) handleNetConn(sess tunnel.Session, msg chan *transport.Message) {
+func (n *network) handleNetConn(s tunnel.Session, msg chan *message) {
 	for {
 		m := new(transport.Message)
-		if err := sess.Recv(m); err != nil {
+		if err := s.Recv(m); err != nil {
 			log.Debugf("Network tunnel [%s] receive error: %v", NetworkChannel, err)
-			if sessErr := sess.Close(); sessErr != nil {
-				log.Debugf("Network tunnel [%s] closing connection error: %v", sessErr)
+			if sessionErr := s.Close(); sessionErr != nil {
+				log.Debugf("Network tunnel [%s] closing connection error: %v", NetworkChannel, sessionErr)
 			}
 			return
 		}
 
 		select {
-		case msg <- m:
+		case msg <- &message{
+			msg:     m,
+			session: s,
+		}:
 		case <-n.closed:
 			return
 		}
@@ -266,7 +276,7 @@ func (n *network) handleNetConn(sess tunnel.Session, msg chan *transport.Message
 }
 
 // acceptNetConn accepts connections from NetworkChannel
-func (n *network) acceptNetConn(l tunnel.Listener, recv chan *transport.Message) {
+func (n *network) acceptNetConn(l tunnel.Listener, recv chan *message) {
 	var i int
 	for {
 		// accept a connection
@@ -295,10 +305,41 @@ func (n *network) acceptNetConn(l tunnel.Listener, recv chan *transport.Message)
 	}
 }
 
+// updatePeerLinks updates link for a given peer
+func (n *network) updatePeerLinks(peerAddr string, linkId string) error {
+	n.Lock()
+	defer n.Unlock()
+	log.Tracef("Network looking up link %s in the peer links", linkId)
+	// lookup the peer link
+	var peerLink tunnel.Link
+	for _, link := range n.tunnel.Links() {
+		if link.Id() == linkId {
+			peerLink = link
+			break
+		}
+	}
+	if peerLink == nil {
+		return ErrPeerLinkNotFound
+	}
+	// if the peerLink is found in the returned links update peerLinks
+	log.Tracef("Network updating peer links for peer %s", peerAddr)
+	// add peerLink to the peerLinks map
+	if link, ok := n.peerLinks[peerAddr]; ok {
+		// if the existing has better Length then the new, replace it
+		if link.Length() < peerLink.Length() {
+			n.peerLinks[peerAddr] = peerLink
+		}
+	} else {
+		n.peerLinks[peerAddr] = peerLink
+	}
+
+	return nil
+}
+
 // processNetChan processes messages received on NetworkChannel
 func (n *network) processNetChan(listener tunnel.Listener) {
 	// receive network message queue
-	recv := make(chan *transport.Message, 128)
+	recv := make(chan *message, 128)
 
 	// accept NetworkChannel connections
 	go n.acceptNetConn(listener, recv)
@@ -307,12 +348,12 @@ func (n *network) processNetChan(listener tunnel.Listener) {
 		select {
 		case m := <-recv:
 			// switch on type of message and take action
-			switch m.Header["Micro-Method"] {
+			switch m.msg.Header["Micro-Method"] {
 			case "connect":
 				// mark the time the message has been received
 				now := time.Now()
 				pbNetConnect := &pbNet.Connect{}
-				if err := proto.Unmarshal(m.Body, pbNetConnect); err != nil {
+				if err := proto.Unmarshal(m.msg.Body, pbNetConnect); err != nil {
 					log.Debugf("Network tunnel [%s] connect unmarshal error: %v", NetworkChannel, err)
 					continue
 				}
@@ -327,6 +368,12 @@ func (n *network) processNetChan(listener tunnel.Listener) {
 					peers:    make(map[string]*node),
 					lastSeen: now,
 				}
+				// update peer links
+				log.Tracef("Network updating peer link %s for peer: %s", m.session.Link(), pbNetConnect.Node.Address)
+				if err := n.updatePeerLinks(pbNetConnect.Node.Address, m.session.Link()); err != nil {
+					log.Debugf("Network failed updating peer links: %s", err)
+				}
+				// add peer to the list of node peers
 				if err := n.node.AddPeer(peer); err == ErrPeerExists {
 					log.Debugf("Network peer exists, refreshing: %s", peer.id)
 					// update lastSeen time for the existing node
@@ -349,7 +396,7 @@ func (n *network) processNetChan(listener tunnel.Listener) {
 				// mark the time the message has been received
 				now := time.Now()
 				pbNetPeer := &pbNet.Peer{}
-				if err := proto.Unmarshal(m.Body, pbNetPeer); err != nil {
+				if err := proto.Unmarshal(m.msg.Body, pbNetPeer); err != nil {
 					log.Debugf("Network tunnel [%s] peer unmarshal error: %v", NetworkChannel, err)
 					continue
 				}
@@ -364,6 +411,11 @@ func (n *network) processNetChan(listener tunnel.Listener) {
 					peers:    make(map[string]*node),
 					lastSeen: now,
 				}
+				// update peer links
+				log.Tracef("Network updating peer link %s for peer: %s", m.session.Link(), pbNetPeer.Node.Address)
+				if err := n.updatePeerLinks(pbNetPeer.Node.Address, m.session.Link()); err != nil {
+					log.Debugf("Network failed updating peer links: %s", err)
+				}
 				if err := n.node.AddPeer(peer); err == nil {
 					// send a solicit message when discovering new peer
 					msg := &pbRtr.Solicit{
@@ -387,13 +439,13 @@ func (n *network) processNetChan(listener tunnel.Listener) {
 
 				// NOTE: we don't unpack MaxDepth toplogy
 				peer = UnpackPeerTopology(pbNetPeer, now, MaxDepth-1)
-				log.Debugf("Network updating topology of node: %s", n.node.id)
+				log.Tracef("Network updating topology of node: %s", n.node.id)
 				if err := n.node.UpdatePeer(peer); err != nil {
 					log.Debugf("Network failed to update peers: %v", err)
 				}
 			case "close":
 				pbNetClose := &pbNet.Close{}
-				if err := proto.Unmarshal(m.Body, pbNetClose); err != nil {
+				if err := proto.Unmarshal(m.msg.Body, pbNetClose); err != nil {
 					log.Debugf("Network tunnel [%s] close unmarshal error: %v", NetworkChannel, err)
 					continue
 				}
@@ -412,6 +464,10 @@ func (n *network) processNetChan(listener tunnel.Listener) {
 				if err := n.prunePeerRoutes(peer); err != nil {
 					log.Debugf("Network failed pruning peer %s routes: %v", peer.id, err)
 				}
+				// deelete peer from the peerLinks
+				n.Lock()
+				delete(n.peerLinks, pbNetClose.Node.Address)
+				n.Unlock()
 			}
 		case <-n.closed:
 			return
@@ -521,6 +577,9 @@ func (n *network) prune() {
 			pruned := n.PruneStalePeerNodes(PruneTime)
 			for id, peer := range pruned {
 				log.Debugf("Network peer exceeded prune time: %s", id)
+				n.Lock()
+				delete(n.peerLinks, peer.address)
+				n.Unlock()
 				if err := n.prunePeerRoutes(peer); err != nil {
 					log.Debugf("Network failed pruning peer %s routes: %v", id, err)
 				}
@@ -528,7 +587,7 @@ func (n *network) prune() {
 			// get a list of all routes
 			routes, err := n.options.Router.Table().List()
 			if err != nil {
-				log.Debugf("Network failed listing routes: %v", err)
+				log.Debugf("Network failed listing routes when pruning peers: %v", err)
 				continue
 			}
 			// collect all the router IDs in the routing table
@@ -549,16 +608,22 @@ func (n *network) prune() {
 }
 
 // handleCtrlConn handles ControlChannel connections
-func (n *network) handleCtrlConn(sess tunnel.Session, msg chan *transport.Message) {
+func (n *network) handleCtrlConn(s tunnel.Session, msg chan *message) {
 	for {
 		m := new(transport.Message)
-		if err := sess.Recv(m); err != nil {
-			log.Debugf("Network tunnel advert receive error: %v", err)
+		if err := s.Recv(m); err != nil {
+			log.Debugf("Network tunnel [%s] receive error: %v", ControlChannel, err)
+			if sessionErr := s.Close(); sessionErr != nil {
+				log.Debugf("Network tunnel [%s] closing connection error: %v", ControlChannel, sessionErr)
+			}
 			return
 		}
 
 		select {
-		case msg <- m:
+		case msg <- &message{
+			msg:     m,
+			session: s,
+		}:
 		case <-n.closed:
 			return
 		}
@@ -566,7 +631,7 @@ func (n *network) handleCtrlConn(sess tunnel.Session, msg chan *transport.Messag
 }
 
 // acceptCtrlConn accepts connections from ControlChannel
-func (n *network) acceptCtrlConn(l tunnel.Listener, recv chan *transport.Message) {
+func (n *network) acceptCtrlConn(l tunnel.Listener, recv chan *message) {
 	var i int
 	for {
 		// accept a connection
@@ -596,42 +661,81 @@ func (n *network) acceptCtrlConn(l tunnel.Listener, recv chan *transport.Message
 	}
 }
 
-// setRouteMetric calculates metric of the route and updates it in place
-// - Local route metric is 1
-// - Routes with ID of adjacent nodes are 10
-// - Routes by peers of the advertiser are 100
-// - Routes beyond your neighbourhood are 1000
-func (n *network) setRouteMetric(route *router.Route) {
+// getHopCount queries network graph and returns hop count for given router
+// - Routes for local services have hop count 1
+// - Routes with ID of adjacent nodes have hop count 2
+// - Routes by peers of the advertiser have hop count 3
+// - Routes beyond node neighbourhood have hop count 4
+func (n *network) getHopCount(rtr string) int {
+	// make sure node.peers are not modified
+	n.node.RLock()
+	defer n.node.RUnlock()
+
 	// we are the origin of the route
-	if route.Router == n.options.Id {
-		route.Metric = 1
-		return
+	if rtr == n.options.Id {
+		return 1
 	}
 
-	// check if the route origin is our peer
-	if _, ok := n.peers[route.Router]; ok {
-		route.Metric = 10
-		return
+	// the route origin is our peer
+	if _, ok := n.peers[rtr]; ok {
+		return 10
 	}
 
-	// check if the route origin is the peer of our peer
+	// the route origin is the peer of our peer
 	for _, peer := range n.peers {
 		for id := range peer.peers {
-			if route.Router == id {
-				route.Metric = 100
-				return
+			if rtr == id {
+				return 100
 			}
 		}
 	}
+	// otherwise we are three hops away
+	return 1000
+}
 
-	// the origin of the route is beyond our neighbourhood
-	route.Metric = 1000
+// getRouteMetric calculates router metric and returns it
+// Route metric is calculated based on link status and route hopd count
+func (n *network) getRouteMetric(router string, gateway string, link string) int64 {
+	// set the route metric
+	n.RLock()
+	defer n.RUnlock()
+
+	if link == "local" && gateway == "" {
+		return 1
+	}
+
+	if link == "local" && gateway != "" {
+		return 2
+	}
+
+	log.Tracef("Network looking up %s link to gateway: %s", link, gateway)
+	if link, ok := n.peerLinks[gateway]; ok {
+		// maka sure delay is non-zero
+		delay := link.Delay()
+		if delay == 0 {
+			delay = 1
+		}
+		// get the route hop count
+		hops := n.getHopCount(router)
+		// make sure length is non-zero
+		length := link.Length()
+		if length == 0 {
+			log.Debugf("Link length is 0 %v %v", link, link.Length())
+			length = 10e9
+		}
+		log.Tracef("Network calculated metric %v delay %v length %v distance %v", (delay*length*int64(hops))/10e6, delay, length, hops)
+		return (delay * length * int64(hops)) / 10e6
+	}
+
+	log.Debugf("Network failed to find a link to gateway: %s", gateway)
+
+	return math.MaxInt64
 }
 
 // processCtrlChan processes messages received on ControlChannel
 func (n *network) processCtrlChan(listener tunnel.Listener) {
 	// receive control message queue
-	recv := make(chan *transport.Message, 128)
+	recv := make(chan *message, 128)
 
 	// accept ControlChannel cconnections
 	go n.acceptCtrlConn(listener, recv)
@@ -640,10 +744,10 @@ func (n *network) processCtrlChan(listener tunnel.Listener) {
 		select {
 		case m := <-recv:
 			// switch on type of message and take action
-			switch m.Header["Micro-Method"] {
+			switch m.msg.Header["Micro-Method"] {
 			case "advert":
 				pbRtrAdvert := &pbRtr.Advert{}
-				if err := proto.Unmarshal(m.Body, pbRtrAdvert); err != nil {
+				if err := proto.Unmarshal(m.msg.Body, pbRtrAdvert); err != nil {
 					log.Debugf("Network fail to unmarshal advert message: %v", err)
 					continue
 				}
@@ -678,17 +782,22 @@ func (n *network) processCtrlChan(listener tunnel.Listener) {
 						Network: event.Route.Network,
 						Router:  event.Route.Router,
 						Link:    event.Route.Link,
-						Metric:  int(event.Route.Metric),
+						Metric:  event.Route.Metric,
 					}
-					// set the route metric
-					n.node.RLock()
-					n.setRouteMetric(&route)
-					n.node.RUnlock()
-					// throw away metric bigger than 1000
-					if route.Metric > 1000 {
-						log.Debugf("Network route metric %d dropping node: %s", route.Metric, route.Router)
-						continue
+					// calculate route metric and add to the advertised metric
+					// we need to make sure we do not overflow math.MaxInt64
+					metric := n.getRouteMetric(event.Route.Router, event.Route.Gateway, event.Route.Link)
+					log.Tracef("Network metric for router %s and gateway %s: %v", event.Route.Router, event.Route.Gateway, metric)
+
+					// check we don't overflow max int 64
+					if d := route.Metric + metric; d > math.MaxInt64 || d <= 0 {
+						// set to max int64 if we overflow
+						route.Metric = math.MaxInt64
+					} else {
+						// set the combined value of metrics otherwise
+						route.Metric = d
 					}
+
 					// create router event
 					e := &router.Event{
 						Type:      router.EventType(event.Type),
@@ -699,7 +808,7 @@ func (n *network) processCtrlChan(listener tunnel.Listener) {
 				}
 				// if no events are eligible for processing continue
 				if len(events) == 0 {
-					log.Debugf("Network no events to be processed by router: %s", n.options.Id)
+					log.Tracef("Network no events to be processed by router: %s", n.options.Id)
 					continue
 				}
 				// create an advert and process it
@@ -717,7 +826,7 @@ func (n *network) processCtrlChan(listener tunnel.Listener) {
 				}
 			case "solicit":
 				pbRtrSolicit := &pbRtr.Solicit{}
-				if err := proto.Unmarshal(m.Body, pbRtrSolicit); err != nil {
+				if err := proto.Unmarshal(m.msg.Body, pbRtrSolicit); err != nil {
 					log.Debugf("Network fail to unmarshal solicit message: %v", err)
 					continue
 				}
@@ -755,12 +864,15 @@ func (n *network) advertise(advertChan <-chan *router.Advert) {
 				if event.Route.Router == advert.Id {
 					// hash the service before advertising it
 					hasher.Reset()
-					hasher.Write([]byte(event.Route.Address + n.node.id))
+					// routes for multiple instances of a service will be collapsed here.
+					// TODO: once we store labels in the table this may need to change
+					// to include the labels in case they differ but highly unlikely
+					hasher.Write([]byte(event.Route.Service + n.node.Address()))
 					address = fmt.Sprintf("%d", hasher.Sum64())
 				}
-
+				// calculate route metric to advertise
+				metric := n.getRouteMetric(event.Route.Router, event.Route.Gateway, event.Route.Link)
 				// NOTE: we override Gateway, Link and Address here
-				// TODO: should we avoid overriding gateway?
 				route := &pbRtr.Route{
 					Service: event.Route.Service,
 					Address: address,
@@ -768,7 +880,7 @@ func (n *network) advertise(advertChan <-chan *router.Advert) {
 					Network: event.Route.Network,
 					Router:  event.Route.Router,
 					Link:    DefaultLink,
-					Metric:  int64(event.Route.Metric),
+					Metric:  metric,
 				}
 				e := &pbRtr.Event{
 					Type:      pbRtr.EventType(event.Type),

network/network.go

@@ -6,6 +6,8 @@ import (
 
 	"github.com/micro/go-micro/client"
 	"github.com/micro/go-micro/server"
+	"github.com/micro/go-micro/transport"
+	"github.com/micro/go-micro/tunnel"
 )
 
 var (
@@ -54,6 +56,14 @@ type Network interface {
 	Server() server.Server
 }
 
+// message is network message
+type message struct {
+	// msg is transport message
+	msg *transport.Message
+	// session is tunnel session
+	session tunnel.Session
+}
+
 // NewNetwork returns a new network interface
 func NewNetwork(opts ...Option) Network {
 	return newNetwork(opts...)

network/resolver/resolver.go

@@ -11,5 +11,6 @@ type Resolver interface {
 
 // A resolved record
 type Record struct {
-	Address string `json:"address"`
+	Address  string `json:"address"`
+	Priority int64  `json:"priority"`
 }

plugin/default.go

@@ -121,5 +121,7 @@ func (p *plugin) Build(path string, c *Config) error {
 		return fmt.Errorf("Failed to create dir %s: %v", filepath.Dir(path), err)
 	}
 	cmd := exec.Command("go", "build", "-buildmode=plugin", "-o", path+".so", goFile)
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
 	return cmd.Run()
 }

proxy/http/http.go

@@ -113,7 +113,7 @@ func (p *Proxy) ServeRequest(ctx context.Context, req server.Request, rsp server
 
 		// set response headers
 		hdr = map[string]string{}
-		for k, _ := range hrsp.Header {
+		for k := range hrsp.Header {
 			hdr[k] = hrsp.Header.Get(k)
 		}
 		// write the header

proxy/mucp/mucp.go

@@ -16,9 +16,11 @@ import (
 	"github.com/micro/go-micro/codec/bytes"
 	"github.com/micro/go-micro/config/options"
 	"github.com/micro/go-micro/errors"
+	"github.com/micro/go-micro/metadata"
 	"github.com/micro/go-micro/proxy"
 	"github.com/micro/go-micro/router"
 	"github.com/micro/go-micro/server"
+	"github.com/micro/go-micro/util/log"
 )
 
 // Proxy will transparently proxy requests to an endpoint.
@@ -42,9 +44,6 @@ type Proxy struct {
 	// A fib of routes service:address
 	sync.RWMutex
 	Routes map[string]map[uint64]router.Route
-
-	// The channel to monitor watcher errors
-	errChan chan error
 }
 
 // read client request and write to server
@@ -95,6 +94,75 @@ func toNodes(routes []router.Route) []string {
 	return nodes
 }
 
+func toSlice(r map[uint64]router.Route) []router.Route {
+	var routes []router.Route
+	for _, v := range r {
+		routes = append(routes, v)
+	}
+
+	// sort the routes in order of metric
+	sort.Slice(routes, func(i, j int) bool { return routes[i].Metric < routes[j].Metric })
+
+	return routes
+}
+
+func (p *Proxy) filterRoutes(ctx context.Context, routes []router.Route) []router.Route {
+	md, ok := metadata.FromContext(ctx)
+	if !ok {
+		return routes
+	}
+
+	var filteredRoutes []router.Route
+
+	// filter the routes based on our headers
+	for _, route := range routes {
+		// process only routes for this id
+		if id := md["Micro-Router"]; len(id) > 0 {
+			if route.Router != id {
+				// skip routes that don't mwatch
+				continue
+			}
+		}
+
+		// only process routes with this network
+		if net := md["Micro-Network"]; len(net) > 0 {
+			if route.Network != net {
+				// skip routes that don't mwatch
+				continue
+			}
+		}
+
+		// process only this gateway
+		if gw := md["Micro-Gateway"]; len(gw) > 0 {
+			// if the gateway matches our address
+			// special case, take the routes with no gateway
+			// TODO: should we strip the gateway from the context?
+			if gw == p.Router.Options().Address {
+				if len(route.Gateway) > 0 && route.Gateway != gw {
+					continue
+				}
+				// otherwise its a local route and we're keeping it
+			} else {
+				// gateway does not match our own
+				if route.Gateway != gw {
+					continue
+				}
+			}
+		}
+
+		// TODO: address based filtering
+		// address := md["Micro-Address"]
+
+		// TODO: label based filtering
+		// requires new field in routing table : route.Labels
+
+		// passed the filter checks
+		filteredRoutes = append(filteredRoutes, route)
+	}
+
+	return filteredRoutes
+}
+
 func (p *Proxy) getLink(r router.Route) (client.Client, error) {
 	if r.Link == "local" || len(p.Links) == 0 {
 		return p.Client, nil
@@ -106,28 +174,27 @@ func (p *Proxy) getLink(r router.Route) (client.Client, error) {
 	return l, nil
 }
 
-func (p *Proxy) getRoute(service string) ([]router.Route, error) {
-	toSlice := func(r map[uint64]router.Route) []router.Route {
-		var routes []router.Route
-		for _, v := range r {
-			routes = append(routes, v)
-		}
-
-		// sort the routes in order of metric
-		sort.Slice(routes, func(i, j int) bool { return routes[i].Metric < routes[j].Metric })
-
-		return routes
-	}
-
+func (p *Proxy) getRoute(ctx context.Context, service string) ([]router.Route, error) {
 	// lookup the route cache first
 	p.Lock()
-	routes, ok := p.Routes[service]
+	cached, ok := p.Routes[service]
 	if ok {
 		p.Unlock()
-		return toSlice(routes), nil
+		routes := toSlice(cached)
+		return p.filterRoutes(ctx, routes), nil
 	}
 	p.Unlock()
 
+	// cache routes for the service
+	routes, err := p.cacheRoutes(service)
+	if err != nil {
+		return nil, err
+	}
+
+	return p.filterRoutes(ctx, routes), nil
+}
+
+func (p *Proxy) cacheRoutes(service string) ([]router.Route, error) {
 	// lookup the routes in the router
 	results, err := p.Router.Lookup(router.QueryService(service))
 	if err != nil {
@@ -148,18 +215,40 @@ func (p *Proxy) getRoute(service string) ([]router.Route, error) {
 		}
 		p.Routes[service][route.Hash()] = route
 	}
-	routes = p.Routes[service]
+	routes := p.Routes[service]
 	p.Unlock()
 
 	return toSlice(routes), nil
 }
 
-// manageRouteCache applies action on a given route to Proxy route cache
-func (p *Proxy) manageRouteCache(route router.Route, action string) error {
+// refreshMetrics will refresh any metrics for our local cached routes.
+// we may not receive new watch events for these as they change.
+func (p *Proxy) refreshMetrics() {
+	var services []string
+
+	// get a list of services to update
+	p.RLock()
+	for service := range p.Routes {
+		services = append(services, service)
+	}
+	p.RUnlock()
+
+	// get and cache the routes for the service
+	for _, service := range services {
+		p.cacheRoutes(service)
+	}
+}
+
+// manageRoutes applies action on a given route to Proxy route cache
+func (p *Proxy) manageRoutes(route router.Route, action string) error {
+	// we only cache what we are actually concerned with
+	p.Lock()
+	defer p.Unlock()
+
 	switch action {
 	case "create", "update":
 		if _, ok := p.Routes[route.Service]; !ok {
-			p.Routes[route.Service] = make(map[uint64]router.Route)
+			return fmt.Errorf("not called %s", route.Service)
 		}
 		p.Routes[route.Service][route.Hash()] = route
 	case "delete":
@@ -173,31 +262,22 @@ func (p *Proxy) manageRouteCache(route router.Route, action string) error {
 
 // watchRoutes watches service routes and updates proxy cache
 func (p *Proxy) watchRoutes() {
-	// this is safe to do as the only way watchRoutes returns is
-	// when some error is written into error channel - we want to bail then
-	defer close(p.errChan)
-
 	// route watcher
 	w, err := p.Router.Watch()
 	if err != nil {
-		p.errChan <- err
 		return
 	}
 
 	for {
 		event, err := w.Next()
 		if err != nil {
-			p.errChan <- err
 			return
 		}
 
-		p.Lock()
-		if err := p.manageRouteCache(event.Route, fmt.Sprintf("%s", event.Type)); err != nil {
+		if err := p.manageRoutes(event.Route, fmt.Sprintf("%s", event.Type)); err != nil {
 			// TODO: should we bail here?
-			p.Unlock()
 			continue
 		}
-		p.Unlock()
 	}
 }
 
@@ -234,7 +314,7 @@ func (p *Proxy) ServeRequest(ctx context.Context, req server.Request, rsp server
 			addresses = []string{p.Endpoint}
 		} else {
 			// get route for endpoint from router
-			addr, err := p.getRoute(p.Endpoint)
+			addr, err := p.getRoute(ctx, p.Endpoint)
 			if err != nil {
 				return err
 			}
@@ -246,7 +326,7 @@ func (p *Proxy) ServeRequest(ctx context.Context, req server.Request, rsp server
 	} else {
 		// no endpoint was specified just lookup the route
 		// get route for endpoint from router
-		addr, err := p.getRoute(service)
+		addr, err := p.getRoute(ctx, service)
 		if err != nil {
 			return err
 		}
@@ -294,6 +374,8 @@ func (p *Proxy) ServeRequest(ctx context.Context, req server.Request, rsp server
 			continue
 		}
 
+		log.Debugf("Proxy using route %+v\n", route)
+
 		// set the address to call
 		addresses := toNodes([]router.Route{route})
 		opts = append(opts, client.WithAddress(addresses...))
@@ -357,39 +439,28 @@ func (p *Proxy) serveRequest(ctx context.Context, link client.Client, service, e
 	// get raw response
 	resp := stream.Response()
 
-	// route watcher error
-	var watchErr error
-
 	// create server response write loop
 	for {
-		select {
-		case err := <-p.errChan:
-			if err != nil {
-				watchErr = err
-			}
-			return watchErr
-		default:
-			// read backend response body
-			body, err := resp.Read()
-			if err == io.EOF {
-				return nil
-			} else if err != nil {
-				return err
-			}
+		// read backend response body
+		body, err := resp.Read()
+		if err == io.EOF {
+			return nil
+		} else if err != nil {
+			return err
+		}
 
-			// read backend response header
-			hdr := resp.Header()
+		// read backend response header
+		hdr := resp.Header()
 
-			// write raw response header to client
-			rsp.WriteHeader(hdr)
+		// write raw response header to client
+		rsp.WriteHeader(hdr)
 
-			// write raw response body to client
-			err = rsp.Write(body)
-			if err == io.EOF {
-				return nil
-			} else if err != nil {
-				return err
-			}
+		// write raw response body to client
+		err = rsp.Write(body)
+		if err == io.EOF {
+			return nil
+		} else if err != nil {
+			return err
 		}
 	}
 
@@ -448,9 +519,6 @@ func NewProxy(opts ...options.Option) proxy.Proxy {
 	// routes cache
 	p.Routes = make(map[string]map[uint64]router.Route)
 
-	// watch router service routes
-	p.errChan = make(chan error, 1)
-
 	go func() {
 		// continuously attempt to watch routes
 		for {
@@ -461,5 +529,19 @@ func NewProxy(opts ...options.Option) proxy.Proxy {
 		}
 	}()
 
+	go func() {
+		t := time.NewTicker(time.Minute)
+		defer t.Stop()
+
+		// we must refresh route metrics since they do not trigger new events
+		for {
+			select {
+			case <-t.C:
+				// refresh route metrics
+				p.refreshMetrics()
+			}
+		}
+	}()
+
 	return p
 }

registry/cache/cache.go

@@ -81,7 +81,7 @@ func (c *cache) isValid(services []*registry.Service, ttl time.Time) bool {
 	}
 
 	// time since ttl is longer than timeout
-	if time.Since(ttl) > c.opts.TTL {
+	if time.Since(ttl) > 0 {
 		return false
 	}
 

registry/consul/consul.go

@@ -1,438 +0,0 @@
-package consul
-
-import (
-	"crypto/tls"
-	"errors"
-	"fmt"
-	"net"
-	"net/http"
-	"runtime"
-	"strconv"
-	"sync"
-	"time"
-
-	consul "github.com/hashicorp/consul/api"
-	"github.com/micro/go-micro/registry"
-	mnet "github.com/micro/go-micro/util/net"
-	hash "github.com/mitchellh/hashstructure"
-)
-
-type consulRegistry struct {
-	Address []string
-	opts    registry.Options
-
-	client *consul.Client
-	config *consul.Config
-
-	// connect enabled
-	connect bool
-
-	queryOptions *consul.QueryOptions
-
-	sync.Mutex
-	register map[string]uint64
-	// lastChecked tracks when a node was last checked as existing in Consul
-	lastChecked map[string]time.Time
-}
-
-func getDeregisterTTL(t time.Duration) time.Duration {
-	// splay slightly for the watcher?
-	splay := time.Second * 5
-	deregTTL := t + splay
-
-	// consul has a minimum timeout on deregistration of 1 minute.
-	if t < time.Minute {
-		deregTTL = time.Minute + splay
-	}
-
-	return deregTTL
-}
-
-func newTransport(config *tls.Config) *http.Transport {
-	if config == nil {
-		config = &tls.Config{
-			InsecureSkipVerify: true,
-		}
-	}
-
-	t := &http.Transport{
-		Proxy: http.ProxyFromEnvironment,
-		Dial: (&net.Dialer{
-			Timeout:   30 * time.Second,
-			KeepAlive: 30 * time.Second,
-		}).Dial,
-		TLSHandshakeTimeout: 10 * time.Second,
-		TLSClientConfig:     config,
-	}
-	runtime.SetFinalizer(&t, func(tr **http.Transport) {
-		(*tr).CloseIdleConnections()
-	})
-	return t
-}
-
-func configure(c *consulRegistry, opts ...registry.Option) {
-	// set opts
-	for _, o := range opts {
-		o(&c.opts)
-	}
-
-	// use default config
-	config := consul.DefaultConfig()
-
-	if c.opts.Context != nil {
-		// Use the consul config passed in the options, if available
-		if co, ok := c.opts.Context.Value("consul_config").(*consul.Config); ok {
-			config = co
-		}
-		if cn, ok := c.opts.Context.Value("consul_connect").(bool); ok {
-			c.connect = cn
-		}
-
-		// Use the consul query options passed in the options, if available
-		if qo, ok := c.opts.Context.Value("consul_query_options").(*consul.QueryOptions); ok && qo != nil {
-			c.queryOptions = qo
-		}
-		if as, ok := c.opts.Context.Value("consul_allow_stale").(bool); ok {
-			c.queryOptions.AllowStale = as
-		}
-	}
-
-	// check if there are any addrs
-	var addrs []string
-
-	// iterate the options addresses
-	for _, address := range c.opts.Addrs {
-		// check we have a port
-		addr, port, err := net.SplitHostPort(address)
-		if ae, ok := err.(*net.AddrError); ok && ae.Err == "missing port in address" {
-			port = "8500"
-			addr = address
-			addrs = append(addrs, net.JoinHostPort(addr, port))
-		} else if err == nil {
-			addrs = append(addrs, net.JoinHostPort(addr, port))
-		}
-	}
-
-	// set the addrs
-	if len(addrs) > 0 {
-		c.Address = addrs
-		config.Address = c.Address[0]
-	}
-
-	if config.HttpClient == nil {
-		config.HttpClient = new(http.Client)
-	}
-
-	// requires secure connection?
-	if c.opts.Secure || c.opts.TLSConfig != nil {
-		config.Scheme = "https"
-		// We're going to support InsecureSkipVerify
-		config.HttpClient.Transport = newTransport(c.opts.TLSConfig)
-	}
-
-	// set timeout
-	if c.opts.Timeout > 0 {
-		config.HttpClient.Timeout = c.opts.Timeout
-	}
-
-	// set the config
-	c.config = config
-
-	// remove client
-	c.client = nil
-
-	// setup the client
-	c.Client()
-}
-
-func (c *consulRegistry) Init(opts ...registry.Option) error {
-	configure(c, opts...)
-	return nil
-}
-
-func (c *consulRegistry) Deregister(s *registry.Service) error {
-	if len(s.Nodes) == 0 {
-		return errors.New("Require at least one node")
-	}
-
-	// delete our hash and time check of the service
-	c.Lock()
-	delete(c.register, s.Name)
-	delete(c.lastChecked, s.Name)
-	c.Unlock()
-
-	node := s.Nodes[0]
-	return c.Client().Agent().ServiceDeregister(node.Id)
-}
-
-func (c *consulRegistry) Register(s *registry.Service, opts ...registry.RegisterOption) error {
-	if len(s.Nodes) == 0 {
-		return errors.New("Require at least one node")
-	}
-
-	var regTCPCheck bool
-	var regInterval time.Duration
-
-	var options registry.RegisterOptions
-	for _, o := range opts {
-		o(&options)
-	}
-
-	if c.opts.Context != nil {
-		if tcpCheckInterval, ok := c.opts.Context.Value("consul_tcp_check").(time.Duration); ok {
-			regTCPCheck = true
-			regInterval = tcpCheckInterval
-		}
-	}
-
-	// create hash of service; uint64
-	h, err := hash.Hash(s, nil)
-	if err != nil {
-		return err
-	}
-
-	// use first node
-	node := s.Nodes[0]
-
-	// get existing hash and last checked time
-	c.Lock()
-	v, ok := c.register[s.Name]
-	lastChecked := c.lastChecked[s.Name]
-	c.Unlock()
-
-	// if it's already registered and matches then just pass the check
-	if ok && v == h {
-		if options.TTL == time.Duration(0) {
-			// ensure that our service hasn't been deregistered by Consul
-			if time.Since(lastChecked) <= getDeregisterTTL(regInterval) {
-				return nil
-			}
-			services, _, err := c.Client().Health().Checks(s.Name, c.queryOptions)
-			if err == nil {
-				for _, v := range services {
-					if v.ServiceID == node.Id {
-						return nil
-					}
-				}
-			}
-		} else {
-			// if the err is nil we're all good, bail out
-			// if not, we don't know what the state is, so full re-register
-			if err := c.Client().Agent().PassTTL("service:"+node.Id, ""); err == nil {
-				return nil
-			}
-		}
-	}
-
-	// encode the tags
-	tags := encodeMetadata(node.Metadata)
-	tags = append(tags, encodeEndpoints(s.Endpoints)...)
-	tags = append(tags, encodeVersion(s.Version)...)
-
-	var check *consul.AgentServiceCheck
-
-	if regTCPCheck {
-		deregTTL := getDeregisterTTL(regInterval)
-
-		check = &consul.AgentServiceCheck{
-			TCP:                            node.Address,
-			Interval:                       fmt.Sprintf("%v", regInterval),
-			DeregisterCriticalServiceAfter: fmt.Sprintf("%v", deregTTL),
-		}
-
-		// if the TTL is greater than 0 create an associated check
-	} else if options.TTL > time.Duration(0) {
-		deregTTL := getDeregisterTTL(options.TTL)
-
-		check = &consul.AgentServiceCheck{
-			TTL:                            fmt.Sprintf("%v", options.TTL),
-			DeregisterCriticalServiceAfter: fmt.Sprintf("%v", deregTTL),
-		}
-	}
-
-	host, pt, _ := net.SplitHostPort(node.Address)
-	if host == "" {
-		host = node.Address
-	}
-	port, _ := strconv.Atoi(pt)
-
-	// register the service
-	asr := &consul.AgentServiceRegistration{
-		ID:      node.Id,
-		Name:    s.Name,
-		Tags:    tags,
-		Port:    port,
-		Address: host,
-		Check:   check,
-	}
-
-	// Specify consul connect
-	if c.connect {
-		asr.Connect = &consul.AgentServiceConnect{
-			Native: true,
-		}
-	}
-
-	if err := c.Client().Agent().ServiceRegister(asr); err != nil {
-		return err
-	}
-
-	// save our hash and time check of the service
-	c.Lock()
-	c.register[s.Name] = h
-	c.lastChecked[s.Name] = time.Now()
-	c.Unlock()
-
-	// if the TTL is 0 we don't mess with the checks
-	if options.TTL == time.Duration(0) {
-		return nil
-	}
-
-	// pass the healthcheck
-	return c.Client().Agent().PassTTL("service:"+node.Id, "")
-}
-
-func (c *consulRegistry) GetService(name string) ([]*registry.Service, error) {
-	var rsp []*consul.ServiceEntry
-	var err error
-
-	// if we're connect enabled only get connect services
-	if c.connect {
-		rsp, _, err = c.Client().Health().Connect(name, "", false, c.queryOptions)
-	} else {
-		rsp, _, err = c.Client().Health().Service(name, "", false, c.queryOptions)
-	}
-	if err != nil {
-		return nil, err
-	}
-
-	serviceMap := map[string]*registry.Service{}
-
-	for _, s := range rsp {
-		if s.Service.Service != name {
-			continue
-		}
-
-		// version is now a tag
-		version, _ := decodeVersion(s.Service.Tags)
-		// service ID is now the node id
-		id := s.Service.ID
-		// key is always the version
-		key := version
-
-		// address is service address
-		address := s.Service.Address
-
-		// use node address
-		if len(address) == 0 {
-			address = s.Node.Address
-		}
-
-		svc, ok := serviceMap[key]
-		if !ok {
-			svc = &registry.Service{
-				Endpoints: decodeEndpoints(s.Service.Tags),
-				Name:      s.Service.Service,
-				Version:   version,
-			}
-			serviceMap[key] = svc
-		}
-
-		var del bool
-
-		for _, check := range s.Checks {
-			// delete the node if the status is critical
-			if check.Status == "critical" {
-				del = true
-				break
-			}
-		}
-
-		// if delete then skip the node
-		if del {
-			continue
-		}
-
-		svc.Nodes = append(svc.Nodes, &registry.Node{
-			Id:       id,
-			Address:  mnet.HostPort(address, s.Service.Port),
-			Metadata: decodeMetadata(s.Service.Tags),
-		})
-	}
-
-	var services []*registry.Service
-	for _, service := range serviceMap {
-		services = append(services, service)
-	}
-	return services, nil
-}
-
-func (c *consulRegistry) ListServices() ([]*registry.Service, error) {
-	rsp, _, err := c.Client().Catalog().Services(c.queryOptions)
-	if err != nil {
-		return nil, err
-	}
-
-	var services []*registry.Service
-
-	for service := range rsp {
-		services = append(services, &registry.Service{Name: service})
-	}
-
-	return services, nil
-}
-
-func (c *consulRegistry) Watch(opts ...registry.WatchOption) (registry.Watcher, error) {
-	return newConsulWatcher(c, opts...)
-}
-
-func (c *consulRegistry) String() string {
-	return "consul"
-}
-
-func (c *consulRegistry) Options() registry.Options {
-	return c.opts
-}
-
-func (c *consulRegistry) Client() *consul.Client {
-	if c.client != nil {
-		return c.client
-	}
-
-	for _, addr := range c.Address {
-		// set the address
-		c.config.Address = addr
-
-		// create a new client
-		tmpClient, _ := consul.NewClient(c.config)
-
-		// test the client
-		_, err := tmpClient.Agent().Host()
-		if err != nil {
-			continue
-		}
-
-		// set the client
-		c.client = tmpClient
-		return c.client
-	}
-
-	// set the default
-	c.client, _ = consul.NewClient(c.config)
-
-	// return the client
-	return c.client
-}
-
-func NewRegistry(opts ...registry.Option) registry.Registry {
-	cr := &consulRegistry{
-		opts:        registry.Options{},
-		register:    make(map[string]uint64),
-		lastChecked: make(map[string]time.Time),
-		queryOptions: &consul.QueryOptions{
-			AllowStale: true,
-		},
-	}
-	configure(cr, opts...)
-	return cr
-}

registry/consul/encoding.go

@@ -1,170 +0,0 @@
-package consul
-
-import (
-	"bytes"
-	"compress/zlib"
-	"encoding/hex"
-	"encoding/json"
-	"io/ioutil"
-
-	"github.com/micro/go-micro/registry"
-)
-
-func encode(buf []byte) string {
-	var b bytes.Buffer
-	defer b.Reset()
-
-	w := zlib.NewWriter(&b)
-	if _, err := w.Write(buf); err != nil {
-		return ""
-	}
-	w.Close()
-
-	return hex.EncodeToString(b.Bytes())
-}
-
-func decode(d string) []byte {
-	hr, err := hex.DecodeString(d)
-	if err != nil {
-		return nil
-	}
-
-	br := bytes.NewReader(hr)
-	zr, err := zlib.NewReader(br)
-	if err != nil {
-		return nil
-	}
-
-	rbuf, err := ioutil.ReadAll(zr)
-	if err != nil {
-		return nil
-	}
-
-	return rbuf
-}
-
-func encodeEndpoints(en []*registry.Endpoint) []string {
-	var tags []string
-	for _, e := range en {
-		if b, err := json.Marshal(e); err == nil {
-			tags = append(tags, "e-"+encode(b))
-		}
-	}
-	return tags
-}
-
-func decodeEndpoints(tags []string) []*registry.Endpoint {
-	var en []*registry.Endpoint
-
-	// use the first format you find
-	var ver byte
-
-	for _, tag := range tags {
-		if len(tag) == 0 || tag[0] != 'e' {
-			continue
-		}
-
-		// check version
-		if ver > 0 && tag[1] != ver {
-			continue
-		}
-
-		var e *registry.Endpoint
-		var buf []byte
-
-		// Old encoding was plain
-		if tag[1] == '=' {
-			buf = []byte(tag[2:])
-		}
-
-		// New encoding is hex
-		if tag[1] == '-' {
-			buf = decode(tag[2:])
-		}
-
-		if err := json.Unmarshal(buf, &e); err == nil {
-			en = append(en, e)
-		}
-
-		// set version
-		ver = tag[1]
-	}
-	return en
-}
-
-func encodeMetadata(md map[string]string) []string {
-	var tags []string
-	for k, v := range md {
-		if b, err := json.Marshal(map[string]string{
-			k: v,
-		}); err == nil {
-			// new encoding
-			tags = append(tags, "t-"+encode(b))
-		}
-	}
-	return tags
-}
-
-func decodeMetadata(tags []string) map[string]string {
-	md := make(map[string]string)
-
-	var ver byte
-
-	for _, tag := range tags {
-		if len(tag) == 0 || tag[0] != 't' {
-			continue
-		}
-
-		// check version
-		if ver > 0 && tag[1] != ver {
-			continue
-		}
-
-		var kv map[string]string
-		var buf []byte
-
-		// Old encoding was plain
-		if tag[1] == '=' {
-			buf = []byte(tag[2:])
-		}
-
-		// New encoding is hex
-		if tag[1] == '-' {
-			buf = decode(tag[2:])
-		}
-
-		// Now unmarshal
-		if err := json.Unmarshal(buf, &kv); err == nil {
-			for k, v := range kv {
-				md[k] = v
-			}
-		}
-
-		// set version
-		ver = tag[1]
-	}
-	return md
-}
-
-func encodeVersion(v string) []string {
-	return []string{"v-" + encode([]byte(v))}
-}
-
-func decodeVersion(tags []string) (string, bool) {
-	for _, tag := range tags {
-		if len(tag) < 2 || tag[0] != 'v' {
-			continue
-		}
-
-		// Old encoding was plain
-		if tag[1] == '=' {
-			return tag[2:], true
-		}
-
-		// New encoding is hex
-		if tag[1] == '-' {
-			return string(decode(tag[2:])), true
-		}
-	}
-	return "", false
-}

registry/consul/encoding_test.go

@@ -1,147 +0,0 @@
-package consul
-
-import (
-	"encoding/json"
-	"testing"
-
-	"github.com/micro/go-micro/registry"
-)
-
-func TestEncodingEndpoints(t *testing.T) {
-	eps := []*registry.Endpoint{
-		&registry.Endpoint{
-			Name: "endpoint1",
-			Request: &registry.Value{
-				Name: "request",
-				Type: "request",
-			},
-			Response: &registry.Value{
-				Name: "response",
-				Type: "response",
-			},
-			Metadata: map[string]string{
-				"foo1": "bar1",
-			},
-		},
-		&registry.Endpoint{
-			Name: "endpoint2",
-			Request: &registry.Value{
-				Name: "request",
-				Type: "request",
-			},
-			Response: &registry.Value{
-				Name: "response",
-				Type: "response",
-			},
-			Metadata: map[string]string{
-				"foo2": "bar2",
-			},
-		},
-		&registry.Endpoint{
-			Name: "endpoint3",
-			Request: &registry.Value{
-				Name: "request",
-				Type: "request",
-			},
-			Response: &registry.Value{
-				Name: "response",
-				Type: "response",
-			},
-			Metadata: map[string]string{
-				"foo3": "bar3",
-			},
-		},
-	}
-
-	testEp := func(ep *registry.Endpoint, enc string) {
-		// encode endpoint
-		e := encodeEndpoints([]*registry.Endpoint{ep})
-
-		// check there are two tags; old and new
-		if len(e) != 1 {
-			t.Fatalf("Expected 1 encoded tags, got %v", e)
-		}
-
-		// check old encoding
-		var seen bool
-
-		for _, en := range e {
-			if en == enc {
-				seen = true
-				break
-			}
-		}
-
-		if !seen {
-			t.Fatalf("Expected %s but not found", enc)
-		}
-
-		// decode
-		d := decodeEndpoints([]string{enc})
-		if len(d) == 0 {
-			t.Fatalf("Expected %v got %v", ep, d)
-		}
-
-		// check name
-		if d[0].Name != ep.Name {
-			t.Fatalf("Expected ep %s got %s", ep.Name, d[0].Name)
-		}
-
-		// check all the metadata exists
-		for k, v := range ep.Metadata {
-			if gv := d[0].Metadata[k]; gv != v {
-				t.Fatalf("Expected key %s val %s got val %s", k, v, gv)
-			}
-		}
-	}
-
-	for _, ep := range eps {
-		// JSON encoded
-		jencoded, err := json.Marshal(ep)
-		if err != nil {
-			t.Fatal(err)
-		}
-
-		// HEX encoded
-		hencoded := encode(jencoded)
-		// endpoint tag
-		hepTag := "e-" + hencoded
-		testEp(ep, hepTag)
-	}
-}
-
-func TestEncodingVersion(t *testing.T) {
-	testData := []struct {
-		decoded string
-		encoded string
-	}{
-		{"1.0.0", "v-789c32d433d03300040000ffff02ce00ee"},
-		{"latest", "v-789cca492c492d2e01040000ffff08cc028e"},
-	}
-
-	for _, data := range testData {
-		e := encodeVersion(data.decoded)
-
-		if e[0] != data.encoded {
-			t.Fatalf("Expected %s got %s", data.encoded, e)
-		}
-
-		d, ok := decodeVersion(e)
-		if !ok {
-			t.Fatalf("Unexpected %t for %s", ok, data.encoded)
-		}
-
-		if d != data.decoded {
-			t.Fatalf("Expected %s got %s", data.decoded, d)
-		}
-
-		d, ok = decodeVersion([]string{data.encoded})
-		if !ok {
-			t.Fatalf("Unexpected %t for %s", ok, data.encoded)
-		}
-
-		if d != data.decoded {
-			t.Fatalf("Expected %s got %s", data.decoded, d)
-		}
-	}
-}

registry/consul/options.go

@@ -1,81 +0,0 @@
-package consul
-
-import (
-	"context"
-	"time"
-
-	consul "github.com/hashicorp/consul/api"
-	"github.com/micro/go-micro/registry"
-)
-
-// Connect specifies services should be registered as Consul Connect services
-func Connect() registry.Option {
-	return func(o *registry.Options) {
-		if o.Context == nil {
-			o.Context = context.Background()
-		}
-		o.Context = context.WithValue(o.Context, "consul_connect", true)
-	}
-}
-
-func Config(c *consul.Config) registry.Option {
-	return func(o *registry.Options) {
-		if o.Context == nil {
-			o.Context = context.Background()
-		}
-		o.Context = context.WithValue(o.Context, "consul_config", c)
-	}
-}
-
-// AllowStale sets whether any Consul server (non-leader) can service
-// a read. This allows for lower latency and higher throughput
-// at the cost of potentially stale data.
-// Works similar to Consul DNS Config option [1].
-// Defaults to true.
-//
-// [1] https://www.consul.io/docs/agent/options.html#allow_stale
-//
-func AllowStale(v bool) registry.Option {
-	return func(o *registry.Options) {
-		if o.Context == nil {
-			o.Context = context.Background()
-		}
-		o.Context = context.WithValue(o.Context, "consul_allow_stale", v)
-	}
-}
-
-// QueryOptions specifies the QueryOptions to be used when calling
-// Consul. See `Consul API` for more information [1].
-//
-// [1] https://godoc.org/github.com/hashicorp/consul/api#QueryOptions
-//
-func QueryOptions(q *consul.QueryOptions) registry.Option {
-	return func(o *registry.Options) {
-		if q == nil {
-			return
-		}
-		if o.Context == nil {
-			o.Context = context.Background()
-		}
-		o.Context = context.WithValue(o.Context, "consul_query_options", q)
-	}
-}
-
-//
-// TCPCheck will tell the service provider to check the service address
-// and port every `t` interval. It will enabled only if `t` is greater than 0.
-// See `TCP + Interval` for more information [1].
-//
-// [1] https://www.consul.io/docs/agent/checks.html
-//
-func TCPCheck(t time.Duration) registry.Option {
-	return func(o *registry.Options) {
-		if t <= time.Duration(0) {
-			return
-		}
-		if o.Context == nil {
-			o.Context = context.Background()
-		}
-		o.Context = context.WithValue(o.Context, "consul_tcp_check", t)
-	}
-}

registry/consul/registry_test.go

@@ -1,208 +0,0 @@
-package consul
-
-import (
-	"bytes"
-	"encoding/json"
-	"errors"
-	"net"
-	"net/http"
-	"testing"
-	"time"
-
-	consul "github.com/hashicorp/consul/api"
-	"github.com/micro/go-micro/registry"
-)
-
-type mockRegistry struct {
-	body   []byte
-	status int
-	err    error
-	url    string
-}
-
-func encodeData(obj interface{}) ([]byte, error) {
-	buf := bytes.NewBuffer(nil)
-	enc := json.NewEncoder(buf)
-	if err := enc.Encode(obj); err != nil {
-		return nil, err
-	}
-	return buf.Bytes(), nil
-}
-
-func newMockServer(rg *mockRegistry, l net.Listener) error {
-	mux := http.NewServeMux()
-	mux.HandleFunc(rg.url, func(w http.ResponseWriter, r *http.Request) {
-		if rg.err != nil {
-			http.Error(w, rg.err.Error(), 500)
-			return
-		}
-		w.WriteHeader(rg.status)
-		w.Write(rg.body)
-	})
-	return http.Serve(l, mux)
-}
-
-func newConsulTestRegistry(r *mockRegistry) (*consulRegistry, func()) {
-	l, err := net.Listen("tcp", "localhost:0")
-	if err != nil {
-		// blurgh?!!
-		panic(err.Error())
-	}
-	cfg := consul.DefaultConfig()
-	cfg.Address = l.Addr().String()
-
-	go newMockServer(r, l)
-
-	var cr = &consulRegistry{
-		config:      cfg,
-		Address:     []string{cfg.Address},
-		opts:        registry.Options{},
-		register:    make(map[string]uint64),
-		lastChecked: make(map[string]time.Time),
-		queryOptions: &consul.QueryOptions{
-			AllowStale: true,
-		},
-	}
-	cr.Client()
-
-	return cr, func() {
-		l.Close()
-	}
-}
-
-func newServiceList(svc []*consul.ServiceEntry) []byte {
-	bts, _ := encodeData(svc)
-	return bts
-}
-
-func TestConsul_GetService_WithError(t *testing.T) {
-	cr, cl := newConsulTestRegistry(&mockRegistry{
-		err: errors.New("client-error"),
-		url: "/v1/health/service/service-name",
-	})
-	defer cl()
-
-	if _, err := cr.GetService("test-service"); err == nil {
-		t.Fatalf("Expected error not to be `nil`")
-	}
-}
-
-func TestConsul_GetService_WithHealthyServiceNodes(t *testing.T) {
-	// warning is still seen as healthy, critical is not
-	svcs := []*consul.ServiceEntry{
-		newServiceEntry(
-			"node-name-1", "node-address-1", "service-name", "v1.0.0",
-			[]*consul.HealthCheck{
-				newHealthCheck("node-name-1", "service-name", "passing"),
-				newHealthCheck("node-name-1", "service-name", "warning"),
-			},
-		),
-		newServiceEntry(
-			"node-name-2", "node-address-2", "service-name", "v1.0.0",
-			[]*consul.HealthCheck{
-				newHealthCheck("node-name-2", "service-name", "passing"),
-				newHealthCheck("node-name-2", "service-name", "warning"),
-			},
-		),
-	}
-
-	cr, cl := newConsulTestRegistry(&mockRegistry{
-		status: 200,
-		body:   newServiceList(svcs),
-		url:    "/v1/health/service/service-name",
-	})
-	defer cl()
-
-	svc, err := cr.GetService("service-name")
-	if err != nil {
-		t.Fatal("Unexpected error", err)
-	}
-
-	if exp, act := 1, len(svc); exp != act {
-		t.Fatalf("Expected len of svc to be `%d`, got `%d`.", exp, act)
-	}
-
-	if exp, act := 2, len(svc[0].Nodes); exp != act {
-		t.Fatalf("Expected len of nodes to be `%d`, got `%d`.", exp, act)
-	}
-}
-
-func TestConsul_GetService_WithUnhealthyServiceNode(t *testing.T) {
-	// warning is still seen as healthy, critical is not
-	svcs := []*consul.ServiceEntry{
-		newServiceEntry(
-			"node-name-1", "node-address-1", "service-name", "v1.0.0",
-			[]*consul.HealthCheck{
-				newHealthCheck("node-name-1", "service-name", "passing"),
-				newHealthCheck("node-name-1", "service-name", "warning"),
-			},
-		),
-		newServiceEntry(
-			"node-name-2", "node-address-2", "service-name", "v1.0.0",
-			[]*consul.HealthCheck{
-				newHealthCheck("node-name-2", "service-name", "passing"),
-				newHealthCheck("node-name-2", "service-name", "critical"),
-			},
-		),
-	}
-
-	cr, cl := newConsulTestRegistry(&mockRegistry{
-		status: 200,
-		body:   newServiceList(svcs),
-		url:    "/v1/health/service/service-name",
-	})
-	defer cl()
-
-	svc, err := cr.GetService("service-name")
-	if err != nil {
-		t.Fatal("Unexpected error", err)
-	}
-
-	if exp, act := 1, len(svc); exp != act {
-		t.Fatalf("Expected len of svc to be `%d`, got `%d`.", exp, act)
-	}
-
-	if exp, act := 1, len(svc[0].Nodes); exp != act {
-		t.Fatalf("Expected len of nodes to be `%d`, got `%d`.", exp, act)
-	}
-}
-
-func TestConsul_GetService_WithUnhealthyServiceNodes(t *testing.T) {
-	// warning is still seen as healthy, critical is not
-	svcs := []*consul.ServiceEntry{
-		newServiceEntry(
-			"node-name-1", "node-address-1", "service-name", "v1.0.0",
-			[]*consul.HealthCheck{
-				newHealthCheck("node-name-1", "service-name", "passing"),
-				newHealthCheck("node-name-1", "service-name", "critical"),
-			},
-		),
-		newServiceEntry(
-			"node-name-2", "node-address-2", "service-name", "v1.0.0",
-			[]*consul.HealthCheck{
-				newHealthCheck("node-name-2", "service-name", "passing"),
-				newHealthCheck("node-name-2", "service-name", "critical"),
-			},
-		),
-	}
-
-	cr, cl := newConsulTestRegistry(&mockRegistry{
-		status: 200,
-		body:   newServiceList(svcs),
-		url:    "/v1/health/service/service-name",
-	})
-	defer cl()
-
-	svc, err := cr.GetService("service-name")
-	if err != nil {
-		t.Fatal("Unexpected error", err)
-	}
-
-	if exp, act := 1, len(svc); exp != act {
-		t.Fatalf("Expected len of svc to be `%d`, got `%d`.", exp, act)
-	}
-
-	if exp, act := 0, len(svc[0].Nodes); exp != act {
-		t.Fatalf("Expected len of nodes to be `%d`, got `%d`.", exp, act)
-	}
-}

registry/consul/watcher.go

@@ -1,290 +0,0 @@
-package consul
-
-import (
-	"fmt"
-	"log"
-	"os"
-	"sync"
-
-	"github.com/hashicorp/consul/api"
-	"github.com/hashicorp/consul/api/watch"
-	"github.com/micro/go-micro/registry"
-)
-
-type consulWatcher struct {
-	r        *consulRegistry
-	wo       registry.WatchOptions
-	wp       *watch.Plan
-	watchers map[string]*watch.Plan
-
-	next chan *registry.Result
-	exit chan bool
-
-	sync.RWMutex
-	services map[string][]*registry.Service
-}
-
-func newConsulWatcher(cr *consulRegistry, opts ...registry.WatchOption) (registry.Watcher, error) {
-	var wo registry.WatchOptions
-	for _, o := range opts {
-		o(&wo)
-	}
-
-	cw := &consulWatcher{
-		r:        cr,
-		wo:       wo,
-		exit:     make(chan bool),
-		next:     make(chan *registry.Result, 10),
-		watchers: make(map[string]*watch.Plan),
-		services: make(map[string][]*registry.Service),
-	}
-
-	wp, err := watch.Parse(map[string]interface{}{"type": "services"})
-	if err != nil {
-		return nil, err
-	}
-
-	wp.Handler = cw.handle
-	go wp.RunWithClientAndLogger(cr.Client(), log.New(os.Stderr, "", log.LstdFlags))
-	cw.wp = wp
-
-	return cw, nil
-}
-
-func (cw *consulWatcher) serviceHandler(idx uint64, data interface{}) {
-	entries, ok := data.([]*api.ServiceEntry)
-	if !ok {
-		return
-	}
-
-	serviceMap := map[string]*registry.Service{}
-	serviceName := ""
-
-	for _, e := range entries {
-		serviceName = e.Service.Service
-		// version is now a tag
-		version, _ := decodeVersion(e.Service.Tags)
-		// service ID is now the node id
-		id := e.Service.ID
-		// key is always the version
-		key := version
-		// address is service address
-		address := e.Service.Address
-
-		// use node address
-		if len(address) == 0 {
-			address = e.Node.Address
-		}
-
-		svc, ok := serviceMap[key]
-		if !ok {
-			svc = &registry.Service{
-				Endpoints: decodeEndpoints(e.Service.Tags),
-				Name:      e.Service.Service,
-				Version:   version,
-			}
-			serviceMap[key] = svc
-		}
-
-		var del bool
-
-		for _, check := range e.Checks {
-			// delete the node if the status is critical
-			if check.Status == "critical" {
-				del = true
-				break
-			}
-		}
-
-		// if delete then skip the node
-		if del {
-			continue
-		}
-
-		svc.Nodes = append(svc.Nodes, &registry.Node{
-			Id:       id,
-			Address:  fmt.Sprintf("%s:%d", address, e.Service.Port),
-			Metadata: decodeMetadata(e.Service.Tags),
-		})
-	}
-
-	cw.RLock()
-	// make a copy
-	rservices := make(map[string][]*registry.Service)
-	for k, v := range cw.services {
-		rservices[k] = v
-	}
-	cw.RUnlock()
-
-	var newServices []*registry.Service
-
-	// serviceMap is the new set of services keyed by name+version
-	for _, newService := range serviceMap {
-		// append to the new set of cached services
-		newServices = append(newServices, newService)
-
-		// check if the service exists in the existing cache
-		oldServices, ok := rservices[serviceName]
-		if !ok {
-			// does not exist? then we're creating brand new entries
-			cw.next <- &registry.Result{Action: "create", Service: newService}
-			continue
-		}
-
-		// service exists. ok let's figure out what to update and delete version wise
-		action := "create"
-
-		for _, oldService := range oldServices {
-			// does this version exist?
-			// no? then default to create
-			if oldService.Version != newService.Version {
-				continue
-			}
-
-			// yes? then it's an update
-			action = "update"
-
-			var nodes []*registry.Node
-			// check the old nodes to see if they've been deleted
-			for _, oldNode := range oldService.Nodes {
-				var seen bool
-				for _, newNode := range newService.Nodes {
-					if newNode.Id == oldNode.Id {
-						seen = true
-						break
-					}
-				}
-				// does the old node exist in the new set of nodes
-				// no? then delete that shit
-				if !seen {
-					nodes = append(nodes, oldNode)
-				}
-			}
-
-			// it's an update rather than creation
-			if len(nodes) > 0 {
-				delService := registry.CopyService(oldService)
-				delService.Nodes = nodes
-				cw.next <- &registry.Result{Action: "delete", Service: delService}
-			}
-		}
-
-		cw.next <- &registry.Result{Action: action, Service: newService}
-	}
-
-	// Now check old versions that may not be in new services map
-	for _, old := range rservices[serviceName] {
-		// old version does not exist in new version map
-		// kill it with fire!
-		if _, ok := serviceMap[old.Version]; !ok {
-			cw.next <- &registry.Result{Action: "delete", Service: old}
-		}
-	}
-
-	cw.Lock()
-	cw.services[serviceName] = newServices
-	cw.Unlock()
-}
-
-func (cw *consulWatcher) handle(idx uint64, data interface{}) {
-	services, ok := data.(map[string][]string)
-	if !ok {
-		return
-	}
-
-	// add new watchers
-	for service, _ := range services {
-		// Filter on watch options
-		// wo.Service: Only watch services we care about
-		if len(cw.wo.Service) > 0 && service != cw.wo.Service {
-			continue
-		}
-
-		if _, ok := cw.watchers[service]; ok {
-			continue
-		}
-		wp, err := watch.Parse(map[string]interface{}{
-			"type":    "service",
-			"service": service,
-		})
-		if err == nil {
-			wp.Handler = cw.serviceHandler
-			go wp.RunWithClientAndLogger(cw.r.Client(), log.New(os.Stderr, "", log.LstdFlags))
-			cw.watchers[service] = wp
-			cw.next <- &registry.Result{Action: "create", Service: &registry.Service{Name: service}}
-		}
-	}
-
-	cw.RLock()
-	// make a copy
-	rservices := make(map[string][]*registry.Service)
-	for k, v := range cw.services {
-		rservices[k] = v
-	}
-	cw.RUnlock()
-
-	// remove unknown services from registry
-	// save the things we want to delete
-	deleted := make(map[string][]*registry.Service)
-
-	for service, _ := range rservices {
-		if _, ok := services[service]; !ok {
-			cw.Lock()
-			// save this before deleting
-			deleted[service] = cw.services[service]
-			delete(cw.services, service)
-			cw.Unlock()
-		}
-	}
-
-	// remove unknown services from watchers
-	for service, w := range cw.watchers {
-		if _, ok := services[service]; !ok {
-			w.Stop()
-			delete(cw.watchers, service)
-			for _, oldService := range deleted[service] {
-				// send a delete for the service nodes that we're removing
-				cw.next <- &registry.Result{Action: "delete", Service: oldService}
-			}
-			// sent the empty list as the last resort to indicate to delete the entire service
-			cw.next <- &registry.Result{Action: "delete", Service: &registry.Service{Name: service}}
-		}
-	}
-}
-
-func (cw *consulWatcher) Next() (*registry.Result, error) {
-	select {
-	case <-cw.exit:
-		return nil, registry.ErrWatcherStopped
-	case r, ok := <-cw.next:
-		if !ok {
-			return nil, registry.ErrWatcherStopped
-		}
-		return r, nil
-	}
-	// NOTE: This is a dead code path: e.g. it will never be reached
-	// as we return in all previous code paths never leading to this return
-	return nil, registry.ErrWatcherStopped
-}
-
-func (cw *consulWatcher) Stop() {
-	select {
-	case <-cw.exit:
-		return
-	default:
-		close(cw.exit)
-		if cw.wp == nil {
-			return
-		}
-		cw.wp.Stop()
-
-		// drain results
-		for {
-			select {
-			case <-cw.next:
-			default:
-				return
-			}
-		}
-	}
-}

registry/consul/watcher_test.go

@@ -1,86 +0,0 @@
-package consul
-
-import (
-	"testing"
-
-	"github.com/hashicorp/consul/api"
-	"github.com/micro/go-micro/registry"
-)
-
-func TestHealthyServiceHandler(t *testing.T) {
-	watcher := newWatcher()
-	serviceEntry := newServiceEntry(
-		"node-name", "node-address", "service-name", "v1.0.0",
-		[]*api.HealthCheck{
-			newHealthCheck("node-name", "service-name", "passing"),
-		},
-	)
-
-	watcher.serviceHandler(1234, []*api.ServiceEntry{serviceEntry})
-
-	if len(watcher.services["service-name"][0].Nodes) != 1 {
-		t.Errorf("Expected length of the service nodes to be 1")
-	}
-}
-
-func TestUnhealthyServiceHandler(t *testing.T) {
-	watcher := newWatcher()
-	serviceEntry := newServiceEntry(
-		"node-name", "node-address", "service-name", "v1.0.0",
-		[]*api.HealthCheck{
-			newHealthCheck("node-name", "service-name", "critical"),
-		},
-	)
-
-	watcher.serviceHandler(1234, []*api.ServiceEntry{serviceEntry})
-
-	if len(watcher.services["service-name"][0].Nodes) != 0 {
-		t.Errorf("Expected length of the service nodes to be 0")
-	}
-}
-
-func TestUnhealthyNodeServiceHandler(t *testing.T) {
-	watcher := newWatcher()
-	serviceEntry := newServiceEntry(
-		"node-name", "node-address", "service-name", "v1.0.0",
-		[]*api.HealthCheck{
-			newHealthCheck("node-name", "service-name", "passing"),
-			newHealthCheck("node-name", "serfHealth", "critical"),
-		},
-	)
-
-	watcher.serviceHandler(1234, []*api.ServiceEntry{serviceEntry})
-
-	if len(watcher.services["service-name"][0].Nodes) != 0 {
-		t.Errorf("Expected length of the service nodes to be 0")
-	}
-}
-
-func newWatcher() *consulWatcher {
-	return &consulWatcher{
-		exit:     make(chan bool),
-		next:     make(chan *registry.Result, 10),
-		services: make(map[string][]*registry.Service),
-	}
-}
-
-func newHealthCheck(node, name, status string) *api.HealthCheck {
-	return &api.HealthCheck{
-		Node:        node,
-		Name:        name,
-		Status:      status,
-		ServiceName: name,
-	}
-}
-
-func newServiceEntry(node, address, name, version string, checks []*api.HealthCheck) *api.ServiceEntry {
-	return &api.ServiceEntry{
-		Node: &api.Node{Node: node, Address: name},
-		Service: &api.AgentService{
-			Service: name,
-			Address: address,
-			Tags:    encodeVersion(version),
-		},
-		Checks: checks,
-	}
-}

registry/encoding_test.go

@@ -6,13 +6,13 @@ import (
 
 func TestEncoding(t *testing.T) {
 	testData := []*mdnsTxt{
-		&mdnsTxt{
+		{
 			Version: "1.0.0",
 			Metadata: map[string]string{
 				"foo": "bar",
 			},
 			Endpoints: []*Endpoint{
-				&Endpoint{
+				{
 					Name: "endpoint1",
 					Request: &Value{
 						Name: "request",

registry/mdns_test.go

@@ -7,11 +7,11 @@ import (
 
 func TestMDNS(t *testing.T) {
 	testData := []*Service{
-		&Service{
+		{
 			Name:    "test1",
 			Version: "1.0.1",
 			Nodes: []*Node{
-				&Node{
+				{
 					Id:      "test1-1",
 					Address: "10.0.0.1:10001",
 					Metadata: map[string]string{
@@ -20,11 +20,11 @@ func TestMDNS(t *testing.T) {
 				},
 			},
 		},
-		&Service{
+		{
 			Name:    "test2",
 			Version: "1.0.2",
 			Nodes: []*Node{
-				&Node{
+				{
 					Id:      "test2-1",
 					Address: "10.0.0.2:10002",
 					Metadata: map[string]string{
@@ -33,11 +33,11 @@ func TestMDNS(t *testing.T) {
 				},
 			},
 		},
-		&Service{
+		{
 			Name:    "test3",
 			Version: "1.0.3",
 			Nodes: []*Node{
-				&Node{
+				{
 					Id:      "test3-1",
 					Address: "10.0.0.3:10003",
 					Metadata: map[string]string{

registry/memory/memory_test.go

@@ -8,7 +8,7 @@ import (
 
 var (
 	testData = map[string][]*registry.Service{
-		"foo": []*registry.Service{
+		"foo": {
 			{
 				Name:    "foo",
 				Version: "1.0.0",
@@ -44,7 +44,7 @@ var (
 				},
 			},
 		},
-		"bar": []*registry.Service{
+		"bar": {
 			{
 				Name:    "bar",
 				Version: "default",

registry/watcher_test.go

@@ -6,11 +6,11 @@ import (
 
 func TestWatcher(t *testing.T) {
 	testData := []*Service{
-		&Service{
+		{
 			Name:    "test1",
 			Version: "1.0.1",
 			Nodes: []*Node{
-				&Node{
+				{
 					Id:      "test1-1",
 					Address: "10.0.0.1:10001",
 					Metadata: map[string]string{
@@ -19,11 +19,11 @@ func TestWatcher(t *testing.T) {
 				},
 			},
 		},
-		&Service{
+		{
 			Name:    "test2",
 			Version: "1.0.2",
 			Nodes: []*Node{
-				&Node{
+				{
 					Id:      "test2-1",
 					Address: "10.0.0.2:10002",
 					Metadata: map[string]string{
@@ -32,11 +32,11 @@ func TestWatcher(t *testing.T) {
 				},
 			},
 		},
-		&Service{
+		{
 			Name:    "test3",
 			Version: "1.0.3",
 			Nodes: []*Node{
-				&Node{
+				{
 					Id:      "test3-1",
 					Address: "10.0.0.3:10003",
 					Metadata: map[string]string{

router/default.go

@@ -23,15 +23,13 @@ const (
 	// AdvertSuppress is advert suppression threshold
 	AdvertSuppress = 2000.0
 	// AdvertRecover is advert recovery threshold
-	AdvertRecover = 750.0
+	AdvertRecover = 500.0
 	// DefaultAdvertTTL is default advertisement TTL
 	DefaultAdvertTTL = 1 * time.Minute
-	// DeletePenalty penalises route deletion
-	DeletePenalty = 1000.0
-	// UpdatePenalty penalises route updates
-	UpdatePenalty = 500.0
+	// Penalty for routes processed multiple times
+	Penalty = 2000.0
 	// PenaltyHalfLife is the time the advert penalty decays to half its value
-	PenaltyHalfLife = 2.0
+	PenaltyHalfLife = 2.5
 	// MaxSuppressTime defines time after which the suppressed advert is deleted
 	MaxSuppressTime = 5 * time.Minute
 )
@@ -385,15 +383,16 @@ func (r *router) advertiseEvents() error {
 				// decay the event penalty
 				delta := time.Since(advert.lastUpdate).Seconds()
 				advert.penalty = advert.penalty * math.Exp(-delta*PenaltyDecay)
-
+				service := advert.event.Route.Service
+				address := advert.event.Route.Address
 				// suppress/recover the event based on its penalty level
 				switch {
 				case advert.penalty > AdvertSuppress && !advert.isSuppressed:
-					log.Debugf("Router supressing advert %d for route %s", key, advert.event.Route.Address)
+					log.Debugf("Router suppressing advert %d %.2f for route %s %s", key, advert.penalty, service, address)
 					advert.isSuppressed = true
 					advert.suppressTime = time.Now()
 				case advert.penalty < AdvertRecover && advert.isSuppressed:
-					log.Debugf("Router recovering advert %d for route %s", key, advert.event.Route.Address)
+					log.Debugf("Router recovering advert %d %.2f for route %s %s", key, advert.penalty, service, address)
 					advert.isSuppressed = false
 				}
 
@@ -425,15 +424,11 @@ func (r *router) advertiseEvents() error {
 			if e == nil {
 				continue
 			}
-			log.Debugf("Router processing table event %s for service %s", e.Type, e.Route.Address)
+			log.Debugf("Router processing table event %s for service %s %s", e.Type, e.Route.Service, e.Route.Address)
+
 			// determine the event penalty
-			var penalty float64
-			switch e.Type {
-			case Update:
-				penalty = UpdatePenalty
-			case Delete:
-				penalty = DeletePenalty
-			}
+			// TODO: should there be any difference in penalty for different event types
+			penalty := Penalty
 
 			// check if we have already registered the route
 			hash := e.Route.Hash()
@@ -456,7 +451,7 @@ func (r *router) advertiseEvents() error {
 			// update event penalty and timestamp
 			advert.lastUpdate = time.Now()
 			advert.penalty += penalty
-			log.Debugf("Router advert %d for route %s event penalty: %f", hash, advert.event.Route.Address, advert.penalty)
+			log.Debugf("Router advert %d for route %s %s event penalty: %f", hash, advert.event.Route.Service, advert.event.Route.Address, advert.penalty)
 		case <-r.exit:
 			// first wait for the advertiser to finish
 			r.advertWg.Wait()
@@ -665,7 +660,7 @@ func (r *router) Process(a *Advert) error {
 		// create a copy of the route
 		route := event.Route
 		action := event.Type
-		log.Debugf("Router %s applying %s from router %s for address: %s", r.options.Id, action, route.Router, route.Address)
+		log.Debugf("Router %s applying %s from router %s for service %s %s", r.options.Id, action, route.Router, route.Service, route.Address)
 		if err := r.manageRoute(route, fmt.Sprintf("%s", action)); err != nil {
 			return fmt.Errorf("failed applying action %s to routing table: %s", action, err)
 		}

router/handler/router.go

@@ -31,7 +31,7 @@ func (r *Router) Lookup(ctx context.Context, req *pb.LookupRequest, resp *pb.Loo
 			Network: route.Network,
 			Router:  route.Router,
 			Link:    route.Link,
-			Metric:  int64(route.Metric),
+			Metric:  route.Metric,
 		}
 		respRoutes = append(respRoutes, respRoute)
 	}
@@ -67,7 +67,7 @@ func (r *Router) Advertise(ctx context.Context, req *pb.Request, stream pb.Route
 				Network: event.Route.Network,
 				Router:  event.Route.Router,
 				Link:    event.Route.Link,
-				Metric:  int64(event.Route.Metric),
+				Metric:  event.Route.Metric,
 			}
 			e := &pb.Event{
 				Type:      pb.EventType(event.Type),
@@ -108,7 +108,7 @@ func (r *Router) Process(ctx context.Context, req *pb.Advert, rsp *pb.ProcessRes
 			Network: event.Route.Network,
 			Router:  event.Route.Router,
 			Link:    event.Route.Link,
-			Metric:  int(event.Route.Metric),
+			Metric:  event.Route.Metric,
 		}
 
 		events[i] = &router.Event{
@@ -174,7 +174,7 @@ func (r *Router) Watch(ctx context.Context, req *pb.WatchRequest, stream pb.Rout
 			Network: event.Route.Network,
 			Router:  event.Route.Router,
 			Link:    event.Route.Link,
-			Metric:  int64(event.Route.Metric),
+			Metric:  event.Route.Metric,
 		}
 
 		tableEvent := &pb.Event{

router/handler/table.go

@@ -20,7 +20,7 @@ func (t *Table) Create(ctx context.Context, route *pb.Route, resp *pb.CreateResp
 		Network: route.Network,
 		Router:  route.Router,
 		Link:    route.Link,
-		Metric:  int(route.Metric),
+		Metric:  route.Metric,
 	})
 	if err != nil {
 		return errors.InternalServerError("go.micro.router", "failed to create route: %s", err)
@@ -37,7 +37,7 @@ func (t *Table) Update(ctx context.Context, route *pb.Route, resp *pb.UpdateResp
 		Network: route.Network,
 		Router:  route.Router,
 		Link:    route.Link,
-		Metric:  int(route.Metric),
+		Metric:  route.Metric,
 	})
 	if err != nil {
 		return errors.InternalServerError("go.micro.router", "failed to update route: %s", err)
@@ -54,7 +54,7 @@ func (t *Table) Delete(ctx context.Context, route *pb.Route, resp *pb.DeleteResp
 		Network: route.Network,
 		Router:  route.Router,
 		Link:    route.Link,
-		Metric:  int(route.Metric),
+		Metric:  route.Metric,
 	})
 	if err != nil {
 		return errors.InternalServerError("go.micro.router", "failed to delete route: %s", err)
@@ -79,7 +79,7 @@ func (t *Table) List(ctx context.Context, req *pb.Request, resp *pb.ListResponse
 			Network: route.Network,
 			Router:  route.Router,
 			Link:    route.Link,
-			Metric:  int64(route.Metric),
+			Metric:  route.Metric,
 		}
 		respRoutes = append(respRoutes, respRoute)
 	}
@@ -104,7 +104,7 @@ func (t *Table) Query(ctx context.Context, req *pb.QueryRequest, resp *pb.QueryR
 			Network: route.Network,
 			Router:  route.Router,
 			Link:    route.Link,
-			Metric:  int64(route.Metric),
+			Metric:  route.Metric,
 		}
 		respRoutes = append(respRoutes, respRoute)
 	}

router/route.go

@@ -8,9 +8,7 @@ var (
 	// DefaultLink is default network link
 	DefaultLink = "local"
 	// DefaultLocalMetric is default route cost for a local route
-	DefaultLocalMetric = 1
-	// DefaultNetworkMetric is default route cost for a network route
-	DefaultNetworkMetric = 10
+	DefaultLocalMetric int64 = 1
 )
 
 // Route is network route
@@ -28,7 +26,7 @@ type Route struct {
 	// Link is network link
 	Link string
 	// Metric is the route cost metric
-	Metric int
+	Metric int64
 }
 
 // Hash returns route hash sum.

router/service/service.go

@@ -132,7 +132,7 @@ func (s *svc) advertiseEvents(advertChan chan *router.Advert, stream pb.Router_A
 				Gateway: event.Route.Gateway,
 				Network: event.Route.Network,
 				Link:    event.Route.Link,
-				Metric:  int(event.Route.Metric),
+				Metric:  event.Route.Metric,
 			}
 
 			events[i] = &router.Event{
@@ -196,7 +196,7 @@ func (s *svc) Process(advert *router.Advert) error {
 			Gateway: event.Route.Gateway,
 			Network: event.Route.Network,
 			Link:    event.Route.Link,
-			Metric:  int64(event.Route.Metric),
+			Metric:  event.Route.Metric,
 		}
 		e := &pb.Event{
 			Type:      pb.EventType(event.Type),
@@ -230,7 +230,7 @@ func (s *svc) Solicit() error {
 
 	// build events to advertise
 	events := make([]*router.Event, len(routes))
-	for i, _ := range events {
+	for i := range events {
 		events[i] = &router.Event{
 			Type:      router.Update,
 			Timestamp: time.Now(),
@@ -346,7 +346,7 @@ func (s *svc) Lookup(q ...router.QueryOption) ([]router.Route, error) {
 			Gateway: route.Gateway,
 			Network: route.Network,
 			Link:    route.Link,
-			Metric:  int(route.Metric),
+			Metric:  route.Metric,
 		}
 	}
 

router/service/table.go

@@ -21,7 +21,7 @@ func (t *table) Create(r router.Route) error {
 		Gateway: r.Gateway,
 		Network: r.Network,
 		Link:    r.Link,
-		Metric:  int64(r.Metric),
+		Metric:  r.Metric,
 	}
 
 	if _, err := t.table.Create(context.Background(), route, t.callOpts...); err != nil {
@@ -39,7 +39,7 @@ func (t *table) Delete(r router.Route) error {
 		Gateway: r.Gateway,
 		Network: r.Network,
 		Link:    r.Link,
-		Metric:  int64(r.Metric),
+		Metric:  r.Metric,
 	}
 
 	if _, err := t.table.Delete(context.Background(), route, t.callOpts...); err != nil {
@@ -57,7 +57,7 @@ func (t *table) Update(r router.Route) error {
 		Gateway: r.Gateway,
 		Network: r.Network,
 		Link:    r.Link,
-		Metric:  int64(r.Metric),
+		Metric:  r.Metric,
 	}
 
 	if _, err := t.table.Update(context.Background(), route, t.callOpts...); err != nil {
@@ -82,7 +82,7 @@ func (t *table) List() ([]router.Route, error) {
 			Gateway: route.Gateway,
 			Network: route.Network,
 			Link:    route.Link,
-			Metric:  int(route.Metric),
+			Metric:  route.Metric,
 		}
 	}
 
@@ -115,7 +115,7 @@ func (t *table) Query(q ...router.QueryOption) ([]router.Route, error) {
 			Gateway: route.Gateway,
 			Network: route.Network,
 			Link:    route.Link,
-			Metric:  int(route.Metric),
+			Metric:  route.Metric,
 		}
 	}
 

router/service/watcher.go

@@ -61,7 +61,7 @@ func (w *watcher) watch(stream pb.Router_WatchService) error {
 			Gateway: resp.Route.Gateway,
 			Network: resp.Route.Network,
 			Link:    resp.Route.Link,
-			Metric:  int(resp.Route.Metric),
+			Metric:  resp.Route.Metric,
 		}
 
 		event := &router.Event{

runtime/default.go

@@ -2,19 +2,18 @@ package runtime
 
 import (
 	"errors"
-	"io"
-	"strings"
+	"fmt"
+	"strconv"
 	"sync"
 	"time"
 
-	"github.com/micro/go-micro/runtime/package"
-	"github.com/micro/go-micro/runtime/process"
-	proc "github.com/micro/go-micro/runtime/process/os"
 	"github.com/micro/go-micro/util/log"
 )
 
 type runtime struct {
 	sync.RWMutex
+	// options configure runtime
+	options Options
 	// used to stop the runtime
 	closed chan bool
 	// used to start new services
@@ -25,158 +24,38 @@ type runtime struct {
 	services map[string]*service
 }
 
-type service struct {
-	sync.RWMutex
+// NewRuntime creates new local runtime and returns it
+func NewRuntime(opts ...Option) Runtime {
+	// get default options
+	options := Options{}
 
-	running bool
-	closed  chan bool
-	err     error
+	// apply requested options
+	for _, o := range opts {
+		o(&options)
+	}
 
-	// output for logs
-	output io.Writer
-
-	// service to manage
-	*Service
-	// process creator
-	Process *proc.Process
-	// Exec
-	Exec *process.Executable
-	// process pid
-	PID *process.PID
-}
-
-func newRuntime() *runtime {
 	return &runtime{
+		options:  options,
 		closed:   make(chan bool),
 		start:    make(chan *service, 128),
 		services: make(map[string]*service),
 	}
 }
 
-func newService(s *Service, c CreateOptions) *service {
-	var exec string
-	var args []string
+// Init initializes runtime options
+func (r *runtime) Init(opts ...Option) error {
+	r.Lock()
+	defer r.Unlock()
 
-	if len(s.Exec) > 0 {
-		parts := strings.Split(s.Exec, " ")
-		exec = parts[0]
-		args = []string{}
-
-		if len(parts) > 1 {
-			args = parts[1:]
-		}
-	} else {
-		// set command
-		exec = c.Command[0]
-		// set args
-		if len(c.Command) > 1 {
-			args = c.Command[1:]
-		}
-	}
-
-	return &service{
-		Service: s,
-		Process: new(proc.Process),
-		Exec: &process.Executable{
-			Binary: &packager.Binary{
-				Name: s.Name,
-				Path: exec,
-			},
-			Env:  c.Env,
-			Args: args,
-		},
-		output: c.Output,
-	}
-}
-
-func (s *service) streamOutput() {
-	go io.Copy(s.output, s.PID.Output)
-	go io.Copy(s.output, s.PID.Error)
-}
-
-func (s *service) Running() bool {
-	s.RLock()
-	defer s.RUnlock()
-	return s.running
-}
-
-func (s *service) Start() error {
-	s.Lock()
-	defer s.Unlock()
-
-	if s.running {
-		return nil
-	}
-
-	// reset
-	s.err = nil
-	s.closed = make(chan bool)
-
-	// TODO: pull source & build binary
-	log.Debugf("Runtime service %s forking new process\n")
-	p, err := s.Process.Fork(s.Exec)
-	if err != nil {
-		return err
-	}
-
-	// set the pid
-	s.PID = p
-	// set to running
-	s.running = true
-
-	if s.output != nil {
-		s.streamOutput()
-	}
-
-	// wait and watch
-	go s.Wait()
-
-	return nil
-}
-
-func (s *service) Stop() error {
-	s.Lock()
-	defer s.Unlock()
-
-	select {
-	case <-s.closed:
-		return nil
-	default:
-		close(s.closed)
-		s.running = false
-		return s.Process.Kill(s.PID)
+	for _, o := range opts {
+		o(&r.options)
 	}
 
 	return nil
 }
 
-func (s *service) Error() error {
-	s.RLock()
-	defer s.RUnlock()
-	return s.err
-}
-
-func (s *service) Wait() {
-	// wait for process to exit
-	err := s.Process.Wait(s.PID)
-
-	s.Lock()
-	defer s.Unlock()
-
-	// save the error
-	if err != nil {
-		s.err = err
-	}
-
-	// no longer running
-	s.running = false
-}
-
-func (r *runtime) run() {
-	r.RLock()
-	closed := r.closed
-	r.RUnlock()
-
+// run runs the runtime management loop
+func (r *runtime) run(events <-chan Event) {
 	t := time.NewTicker(time.Second * 5)
 	defer t.Stop()
 
@@ -201,19 +80,67 @@ func (r *runtime) run() {
 			if service.Running() {
 				continue
 			}
-
 			// TODO: check service error
-			log.Debugf("Starting %s", service.Name)
+			log.Debugf("Runtime starting service %s", service.Name)
 			if err := service.Start(); err != nil {
-				log.Debugf("Runtime error starting %s: %v", service.Name, err)
+				log.Debugf("Runtime error starting service %s: %v", service.Name, err)
 			}
-		case <-closed:
-			// TODO: stop all the things
+		case event := <-events:
+			log.Debugf("Runtime received notification event: %v", event)
+			// NOTE: we only handle Update events for now
+			switch event.Type {
+			case Update:
+				// parse returned response to timestamp
+				updateTimeStamp, err := strconv.ParseInt(event.Version, 10, 64)
+				if err != nil {
+					log.Debugf("Runtime error parsing update build time: %v", err)
+					continue
+				}
+				buildTime := time.Unix(updateTimeStamp, 0)
+				processEvent := func(event Event, service *Service) error {
+					buildTimeStamp, err := strconv.ParseInt(service.Version, 10, 64)
+					if err != nil {
+						return err
+					}
+					muBuild := time.Unix(buildTimeStamp, 0)
+					if buildTime.After(muBuild) {
+						if err := r.Update(service); err != nil {
+							return err
+						}
+						service.Version = fmt.Sprintf("%d", buildTime.Unix())
+					}
+					return nil
+				}
+				r.Lock()
+				if len(event.Service) > 0 {
+					service, ok := r.services[event.Service]
+					if !ok {
+						log.Debugf("Runtime unknown service: %s", event.Service)
+						r.Unlock()
+						continue
+					}
+					if err := processEvent(event, service.Service); err != nil {
+						log.Debugf("Runtime error updating service %s: %v", event.Service, err)
+					}
+					r.Unlock()
+					continue
+				}
+				// if blank service was received we update all services
+				for _, service := range r.services {
+					if err := processEvent(event, service.Service); err != nil {
+						log.Debugf("Runtime error updating service %s: %v", service.Name, err)
+					}
+				}
+				r.Unlock()
+			}
+		case <-r.closed:
+			log.Debugf("Runtime stopped.")
 			return
 		}
 	}
 }
 
+// Create creates a new service which is then started by runtime
 func (r *runtime) Create(s *Service, opts ...CreateOption) error {
 	r.Lock()
 	defer r.Unlock()
@@ -240,6 +167,7 @@ func (r *runtime) Create(s *Service, opts ...CreateOption) error {
 	return nil
 }
 
+// Delete removes the service from the runtime and stops it
 func (r *runtime) Delete(s *Service) error {
 	r.Lock()
 	defer r.Unlock()
@@ -252,6 +180,31 @@ func (r *runtime) Delete(s *Service) error {
 	return nil
 }
 
+// Update attemps to update the service
+func (r *runtime) Update(s *Service) error {
+	// delete the service
+	if err := r.Delete(s); err != nil {
+		return err
+	}
+
+	// create new service
+	return r.Create(s)
+}
+
+// List returns a slice of all services tracked by the runtime
+func (r *runtime) List() ([]*Service, error) {
+	var services []*Service
+	r.RLock()
+	defer r.RUnlock()
+
+	for _, service := range r.services {
+		services = append(services, service.Service)
+	}
+
+	return services, nil
+}
+
+// Start starts the runtime
 func (r *runtime) Start() error {
 	r.Lock()
 	defer r.Unlock()
@@ -265,11 +218,22 @@ func (r *runtime) Start() error {
 	r.running = true
 	r.closed = make(chan bool)
 
-	go r.run()
+	var events <-chan Event
+	if r.options.Notifier != nil {
+		var err error
+		events, err = r.options.Notifier.Notify()
+		if err != nil {
+			// TODO: should we bail here?
+			log.Debugf("Runtime failed to start update notifier")
+		}
+	}
+
+	go r.run(events)
 
 	return nil
 }
 
+// Stop stops the runtime
 func (r *runtime) Stop() error {
 	r.Lock()
 	defer r.Unlock()
@@ -292,7 +256,16 @@ func (r *runtime) Stop() error {
 			log.Debugf("Runtime stopping %s", service.Name)
 			service.Stop()
 		}
+		// stop the notifier too
+		if r.options.Notifier != nil {
+			return r.options.Notifier.Close()
+		}
 	}
 
 	return nil
 }
+
+// String implements stringer interface
+func (r *runtime) String() string {
+	return "local"
+}

runtime/kubernetes/client/api/request.go

@@ -0,0 +1,223 @@
+package api
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io"
+	"net/http"
+	"net/url"
+
+	"github.com/micro/go-micro/runtime/kubernetes/client/watch"
+	"github.com/micro/go-micro/util/log"
+)
+
+// Request is used to construct a http request for the k8s API.
+type Request struct {
+	client    *http.Client
+	header    http.Header
+	params    url.Values
+	method    string
+	host      string
+	namespace string
+
+	resource     string
+	resourceName *string
+	body         io.Reader
+
+	err error
+}
+
+// Params is the object to pass in to set paramaters
+// on a request.
+type Params struct {
+	LabelSelector map[string]string
+	Annotations   map[string]string
+	Watch         bool
+}
+
+// verb sets method
+func (r *Request) verb(method string) *Request {
+	r.method = method
+	return r
+}
+
+// Get request
+func (r *Request) Get() *Request {
+	return r.verb("GET")
+}
+
+// Post request
+func (r *Request) Post() *Request {
+	return r.verb("POST")
+}
+
+// Put request
+func (r *Request) Put() *Request {
+	return r.verb("PUT")
+}
+
+// Patch request
+// https://github.com/kubernetes/kubernetes/blob/master/docs/devel/api-conventions.md#patch-operations
+func (r *Request) Patch() *Request {
+	return r.verb("PATCH").SetHeader("Content-Type", "application/strategic-merge-patch+json")
+}
+
+// Delete request
+func (r *Request) Delete() *Request {
+	return r.verb("DELETE")
+}
+
+// Namespace is to set the namespace to operate on
+func (r *Request) Namespace(s string) *Request {
+	r.namespace = s
+	return r
+}
+
+// Resource is the type of resource the operation is
+// for, such as "services", "endpoints" or "pods"
+func (r *Request) Resource(s string) *Request {
+	r.resource = s
+	return r
+}
+
+// Name is for targeting a specific resource by id
+func (r *Request) Name(s string) *Request {
+	r.resourceName = &s
+	return r
+}
+
+// Body pass in a body to set, this is for POST, PUT
+// and PATCH requests
+func (r *Request) Body(in interface{}) *Request {
+	b := new(bytes.Buffer)
+	if err := json.NewEncoder(b).Encode(&in); err != nil {
+		r.err = err
+		return r
+	}
+	log.Debugf("Patch body: %v", b)
+	r.body = b
+	return r
+}
+
+// Params isused to set paramters on a request
+func (r *Request) Params(p *Params) *Request {
+	for k, v := range p.LabelSelector {
+		r.params.Add("labelSelectors", k+"="+v)
+	}
+
+	return r
+}
+
+// SetHeader sets a header on a request with
+// a `key` and `value`
+func (r *Request) SetHeader(key, value string) *Request {
+	r.header.Add(key, value)
+	return r
+}
+
+// request builds the http.Request from the options
+func (r *Request) request() (*http.Request, error) {
+	var url string
+	switch r.resource {
+	case "pods":
+		// /api/v1/namespaces/{namespace}/pods
+		url = fmt.Sprintf("%s/api/v1/namespaces/%s/%s/", r.host, r.namespace, r.resource)
+	case "deployments":
+		// /apis/apps/v1/namespaces/{namespace}/deployments/{name}
+		url = fmt.Sprintf("%s/apis/apps/v1/namespaces/%s/%s/", r.host, r.namespace, r.resource)
+	}
+
+	// append resourceName if it is present
+	if r.resourceName != nil {
+		url += *r.resourceName
+	}
+
+	// append any query params
+	if len(r.params) > 0 {
+		url += "?" + r.params.Encode()
+	}
+
+	// build request
+	req, err := http.NewRequest(r.method, url, r.body)
+	if err != nil {
+		return nil, err
+	}
+
+	// set headers on request
+	req.Header = r.header
+	return req, nil
+}
+
+// Do builds and triggers the request
+func (r *Request) Do() *Response {
+	if r.err != nil {
+		return &Response{
+			err: r.err,
+		}
+	}
+
+	req, err := r.request()
+	if err != nil {
+		return &Response{
+			err: err,
+		}
+	}
+
+	log.Debugf("kubernetes api request: %v", req)
+
+	res, err := r.client.Do(req)
+	if err != nil {
+		return &Response{
+			err: err,
+		}
+	}
+
+	log.Debugf("kubernetes api response: %v", res)
+
+	// return res, err
+	return newResponse(res, err)
+}
+
+// Watch builds and triggers the request, but
+// will watch instead of return an object
+func (r *Request) Watch() (watch.Watch, error) {
+	if r.err != nil {
+		return nil, r.err
+	}
+
+	r.params.Set("watch", "true")
+
+	req, err := r.request()
+	if err != nil {
+		return nil, err
+	}
+
+	w, err := watch.NewBodyWatcher(req, r.client)
+	return w, err
+}
+
+// Options ...
+type Options struct {
+	Host        string
+	Namespace   string
+	BearerToken *string
+	Client      *http.Client
+}
+
+// NewRequest creates a k8s api request
+func NewRequest(opts *Options) *Request {
+	req := &Request{
+		header:    make(http.Header),
+		params:    make(url.Values),
+		client:    opts.Client,
+		namespace: opts.Namespace,
+		host:      opts.Host,
+	}
+
+	if opts.BearerToken != nil {
+		req.SetHeader("Authorization", "Bearer "+*opts.BearerToken)
+	}
+
+	return req
+}

runtime/kubernetes/client/api/response.go

@@ -0,0 +1,94 @@
+package api
+
+import (
+	"encoding/json"
+	"errors"
+	"io/ioutil"
+	"net/http"
+
+	"github.com/micro/go-micro/util/log"
+)
+
+// Errors ...
+var (
+	ErrNotFound = errors.New("kubernetes: not found")
+	ErrDecode   = errors.New("kubernetes: error decoding")
+	ErrOther    = errors.New("kubernetes: unknown error")
+)
+
+// Status is an object that is returned when a request
+// failed or delete succeeded.
+// type Status struct {
+// 	Kind    string `json:"kind"`
+// 	Status  string `json:"status"`
+// 	Message string `json:"message"`
+// 	Reason  string `json:"reason"`
+// 	Code    int    `json:"code"`
+// }
+
+// Response ...
+type Response struct {
+	res *http.Response
+	err error
+
+	body []byte
+}
+
+// Error returns an error
+func (r *Response) Error() error {
+	return r.err
+}
+
+// StatusCode returns status code for response
+func (r *Response) StatusCode() int {
+	return r.res.StatusCode
+}
+
+// Into decode body into `data`
+func (r *Response) Into(data interface{}) error {
+	if r.err != nil {
+		return r.err
+	}
+
+	defer r.res.Body.Close()
+	decoder := json.NewDecoder(r.res.Body)
+	err := decoder.Decode(&data)
+	if err != nil {
+		return ErrDecode
+	}
+
+	return r.err
+}
+
+func newResponse(res *http.Response, err error) *Response {
+	r := &Response{
+		res: res,
+		err: err,
+	}
+
+	if err != nil {
+		return r
+	}
+
+	if r.res.StatusCode == http.StatusOK ||
+		r.res.StatusCode == http.StatusCreated ||
+		r.res.StatusCode == http.StatusNoContent {
+		// Non error status code
+		return r
+	}
+
+	if r.res.StatusCode == http.StatusNotFound {
+		r.err = ErrNotFound
+		return r
+	}
+
+	log.Logf("kubernetes: request failed with code %v", r.res.StatusCode)
+
+	b, err := ioutil.ReadAll(r.res.Body)
+	if err == nil {
+		log.Log("kubernetes: request failed with body:")
+		log.Log(string(b))
+	}
+	r.err = ErrOther
+	return r
+}

runtime/kubernetes/client/client.go

@@ -0,0 +1,102 @@
+package client
+
+import (
+	"crypto/tls"
+	"errors"
+	"io/ioutil"
+	"net/http"
+	"os"
+	"path"
+
+	"github.com/micro/go-micro/runtime/kubernetes/client/api"
+	"github.com/micro/go-micro/util/log"
+)
+
+var (
+	serviceAccountPath = "/var/run/secrets/kubernetes.io/serviceaccount"
+	// ErrReadNamespace is returned when the names could not be read from service account
+	ErrReadNamespace = errors.New("Could not read namespace from service account secret")
+)
+
+// Client ...
+type client struct {
+	opts *api.Options
+}
+
+// NewClientInCluster should work similarily to the official api
+// NewInClient by setting up a client configuration for use within
+// a k8s pod.
+func NewClientInCluster() *client {
+	host := "https://" + os.Getenv("KUBERNETES_SERVICE_HOST") + ":" + os.Getenv("KUBERNETES_SERVICE_PORT")
+
+	s, err := os.Stat(serviceAccountPath)
+	if err != nil {
+		log.Fatal(err)
+	}
+	if s == nil || !s.IsDir() {
+		log.Fatal(errors.New("no k8s service account found"))
+	}
+
+	token, err := ioutil.ReadFile(path.Join(serviceAccountPath, "token"))
+	if err != nil {
+		log.Fatal(err)
+	}
+	t := string(token)
+
+	ns, err := detectNamespace()
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	crt, err := CertPoolFromFile(path.Join(serviceAccountPath, "ca.crt"))
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	c := &http.Client{
+		Transport: &http.Transport{
+			TLSClientConfig: &tls.Config{
+				RootCAs: crt,
+			},
+			DisableCompression: true,
+		},
+	}
+
+	return &client{
+		opts: &api.Options{
+			Client:      c,
+			Host:        host,
+			Namespace:   ns,
+			BearerToken: &t,
+		},
+	}
+}
+
+func detectNamespace() (string, error) {
+	nsPath := path.Join(serviceAccountPath, "namespace")
+
+	// Make sure it's a file and we can read it
+	if s, e := os.Stat(nsPath); e != nil {
+		return "", e
+	} else if s.IsDir() {
+		return "", ErrReadNamespace
+	}
+
+	// Read the file, and cast to a string
+	if ns, e := ioutil.ReadFile(nsPath); e != nil {
+		return string(ns), e
+	} else {
+		return string(ns), nil
+	}
+}
+
+// UpdateDeployment
+func (c *client) UpdateDeployment(name string, body interface{}) error {
+	return api.NewRequest(c.opts).
+		Patch().
+		Resource("deployments").
+		Name(name).
+		Body(body).
+		Do().
+		Error()
+}

runtime/kubernetes/client/kubernetes.go

@@ -0,0 +1,12 @@
+package client
+
+// Kubernetes client
+type Kubernetes interface {
+	// UpdateDeployment patches deployment annotations with new metadata
+	UpdateDeployment(string, interface{}) error
+}
+
+// Metadata defines api request metadata
+type Metadata struct {
+	Annotations map[string]string `json:"annotations,omitempty"`
+}

runtime/kubernetes/client/utils.go

@@ -0,0 +1,74 @@
+package client
+
+import (
+	"crypto/x509"
+	"encoding/pem"
+	"errors"
+	"fmt"
+	"io/ioutil"
+)
+
+// COPIED FROM
+// https://github.com/kubernetes/kubernetes/blob/7a725418af4661067b56506faabc2d44c6d7703a/pkg/util/crypto/crypto.go
+
+// CertPoolFromFile returns an x509.CertPool containing the certificates in the given PEM-encoded file.
+// Returns an error if the file could not be read, a certificate could not be parsed, or if the file does not contain any certificates
+func CertPoolFromFile(filename string) (*x509.CertPool, error) {
+	certs, err := certificatesFromFile(filename)
+	if err != nil {
+		return nil, err
+	}
+	pool := x509.NewCertPool()
+	for _, cert := range certs {
+		pool.AddCert(cert)
+	}
+	return pool, nil
+}
+
+// certificatesFromFile returns the x509.Certificates contained in the given PEM-encoded file.
+// Returns an error if the file could not be read, a certificate could not be parsed, or if the file does not contain any certificates
+func certificatesFromFile(file string) ([]*x509.Certificate, error) {
+	if len(file) == 0 {
+		return nil, errors.New("error reading certificates from an empty filename")
+	}
+	pemBlock, err := ioutil.ReadFile(file)
+	if err != nil {
+		return nil, err
+	}
+	certs, err := CertsFromPEM(pemBlock)
+	if err != nil {
+		return nil, fmt.Errorf("error reading %s: %s", file, err)
+	}
+	return certs, nil
+}
+
+// CertsFromPEM returns the x509.Certificates contained in the given PEM-encoded byte array
+// Returns an error if a certificate could not be parsed, or if the data does not contain any certificates
+func CertsFromPEM(pemCerts []byte) ([]*x509.Certificate, error) {
+	ok := false
+	certs := []*x509.Certificate{}
+	for len(pemCerts) > 0 {
+		var block *pem.Block
+		block, pemCerts = pem.Decode(pemCerts)
+		if block == nil {
+			break
+		}
+		// Only use PEM "CERTIFICATE" blocks without extra headers
+		if block.Type != "CERTIFICATE" || len(block.Headers) != 0 {
+			continue
+		}
+
+		cert, err := x509.ParseCertificate(block.Bytes)
+		if err != nil {
+			return certs, err
+		}
+
+		certs = append(certs, cert)
+		ok = true
+	}
+
+	if !ok {
+		return certs, errors.New("could not read any certificates")
+	}
+	return certs, nil
+}

runtime/kubernetes/client/watch/body.go

@@ -0,0 +1,92 @@
+package watch
+
+import (
+	"bufio"
+	"encoding/json"
+	"net/http"
+	"time"
+)
+
+// bodyWatcher scans the body of a request for chunks
+type bodyWatcher struct {
+	results chan Event
+	stop    chan struct{}
+	res     *http.Response
+	req     *http.Request
+}
+
+// Changes returns the results channel
+func (wr *bodyWatcher) ResultChan() <-chan Event {
+	return wr.results
+}
+
+// Stop cancels the request
+func (wr *bodyWatcher) Stop() {
+	select {
+	case <-wr.stop:
+		return
+	default:
+		close(wr.stop)
+		close(wr.results)
+	}
+}
+
+func (wr *bodyWatcher) stream() {
+	reader := bufio.NewReader(wr.res.Body)
+
+	// ignore first few messages from stream,
+	// as they are usually old.
+	ignore := true
+
+	go func() {
+		<-time.After(time.Second)
+		ignore = false
+	}()
+
+	go func() {
+		// stop the watcher
+		defer wr.Stop()
+
+		for {
+			// read a line
+			b, err := reader.ReadBytes('\n')
+			if err != nil {
+				return
+			}
+
+			// ignore for the first second
+			if ignore {
+				continue
+			}
+
+			// send the event
+			var event Event
+			if err := json.Unmarshal(b, &event); err != nil {
+				continue
+			}
+			wr.results <- event
+		}
+	}()
+}
+
+// NewBodyWatcher creates a k8s body watcher for
+// a given http request
+func NewBodyWatcher(req *http.Request, client *http.Client) (Watch, error) {
+	stop := make(chan struct{})
+	req.Cancel = stop
+
+	res, err := client.Do(req)
+	if err != nil {
+		return nil, err
+	}
+
+	wr := &bodyWatcher{
+		results: make(chan Event),
+		stop:    stop,
+		req:     req,
+		res:     res,
+	}
+
+	go wr.stream()
+	return wr, nil
+}

runtime/kubernetes/client/watch/watch.go

@@ -0,0 +1,26 @@
+package watch
+
+import "encoding/json"
+
+// Watch ...
+type Watch interface {
+	Stop()
+	ResultChan() <-chan Event
+}
+
+// EventType defines the possible types of events.
+type EventType string
+
+// EventTypes used
+const (
+	Added    EventType = "ADDED"
+	Modified EventType = "MODIFIED"
+	Deleted  EventType = "DELETED"
+	Error    EventType = "ERROR"
+)
+
+// Event represents a single event to a watched resource.
+type Event struct {
+	Type   EventType       `json:"type"`
+	Object json.RawMessage `json:"object"`
+}

runtime/kubernetes/client/watch/watch_test.go

@@ -0,0 +1,71 @@
+package watch
+
+import (
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+	"time"
+)
+
+var actions = []string{
+	`{"type": "create", "object":{"foo": "bar"}}`,
+	`{"type": "delete", INVALID}`,
+	`{"type": "update", "object":{"foo": {"foo": "bar"}}}`,
+	`{"type": "delete", "object":null}`,
+}
+
+func TestBodyWatcher(t *testing.T) {
+	// set up server with handler to flush strings from ch.
+	ch := make(chan string)
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		flusher, ok := w.(http.Flusher)
+		if !ok {
+			t.Fatal("expected ResponseWriter to be a flusher")
+		}
+
+		fmt.Fprintf(w, "\n")
+		flusher.Flush()
+
+		for v := range ch {
+			fmt.Fprintf(w, "%s\n", v)
+			flusher.Flush()
+			time.Sleep(10 * time.Millisecond)
+		}
+	}))
+	defer ts.Close()
+
+	req, err := http.NewRequest("GET", ts.URL, nil)
+	if err != nil {
+		t.Fatalf("did not expect NewRequest to return err: %v", err)
+	}
+
+	// setup body watcher
+	w, err := NewBodyWatcher(req, http.DefaultClient)
+	if err != nil {
+		t.Fatalf("did not expect NewBodyWatcher to return %v", err)
+	}
+
+	<-time.After(time.Second)
+
+	// send action strings in, and expect result back
+	ch <- actions[0]
+	if r := <-w.ResultChan(); r.Type != "create" {
+		t.Fatalf("expected result to be create")
+	}
+
+	ch <- actions[1] // should be ignored as its invalid json
+	ch <- actions[2]
+	if r := <-w.ResultChan(); r.Type != "update" {
+		t.Fatalf("expected result to be update")
+	}
+
+	ch <- actions[3]
+	if r := <-w.ResultChan(); r.Type != "delete" {
+		t.Fatalf("expected result to be delete")
+	}
+
+	// stop should clean up all channels.
+	w.Stop()
+	close(ch)
+}

runtime/kubernetes/kubernetes.go

@@ -0,0 +1,290 @@
+// Package kubernetes implements kubernetes micro runtime
+package kubernetes
+
+import (
+	"errors"
+	"fmt"
+	"strconv"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/micro/go-micro/runtime"
+	"github.com/micro/go-micro/runtime/kubernetes/client"
+	"github.com/micro/go-micro/util/log"
+)
+
+type kubernetes struct {
+	sync.RWMutex
+	// options configure runtime
+	options runtime.Options
+	// indicates if we're running
+	running bool
+	// used to start new services
+	start chan *runtime.Service
+	// used to stop the runtime
+	closed chan bool
+	// service tracks deployed services
+	services map[string]*runtime.Service
+	// client is kubernetes client
+	client client.Kubernetes
+}
+
+// NewRuntime creates new kubernetes runtime
+func NewRuntime(opts ...runtime.Option) runtime.Runtime {
+	// get default options
+	options := runtime.Options{}
+
+	// apply requested options
+	for _, o := range opts {
+		o(&options)
+	}
+
+	// kubernetes client
+	client := client.NewClientInCluster()
+
+	return &kubernetes{
+		options:  options,
+		closed:   make(chan bool),
+		start:    make(chan *runtime.Service, 128),
+		services: make(map[string]*runtime.Service),
+		client:   client,
+	}
+}
+
+// Init initializes runtime options
+func (k *kubernetes) Init(opts ...runtime.Option) error {
+	k.Lock()
+	defer k.Unlock()
+
+	for _, o := range opts {
+		o(&k.options)
+	}
+
+	return nil
+}
+
+// Registers a service
+func (k *kubernetes) Create(s *runtime.Service, opts ...runtime.CreateOption) error {
+	k.Lock()
+	defer k.Unlock()
+
+	// TODO:
+	// * create service
+	// * create deployment
+
+	// NOTE: our services have micro- prefix
+	muName := strings.Split(s.Name, ".")
+	s.Name = "micro-" + muName[len(muName)-1]
+
+	// NOTE: we are tracking this in memory for now
+	if _, ok := k.services[s.Name]; ok {
+		return errors.New("service already registered")
+	}
+
+	var options runtime.CreateOptions
+	for _, o := range opts {
+		o(&options)
+	}
+
+	// save service
+	k.services[s.Name] = s
+	// push into start queue
+	k.start <- k.services[s.Name]
+
+	return nil
+}
+
+// Remove a service
+func (k *kubernetes) Delete(s *runtime.Service) error {
+	k.Lock()
+	defer k.Unlock()
+
+	// TODO:
+	// * delete service
+	// * delete dpeloyment
+
+	// NOTE: we are tracking this in memory for now
+	if s, ok := k.services[s.Name]; ok {
+		delete(k.services, s.Name)
+		return nil
+	}
+
+	return nil
+}
+
+// Update the service in place
+func (k *kubernetes) Update(s *runtime.Service) error {
+	type body struct {
+		Metadata *client.Metadata `json:"metadata"`
+	}
+	// parse version into human readable timestamp
+	updateTimeStamp, err := strconv.ParseInt(s.Version, 10, 64)
+	if err != nil {
+		return err
+	}
+	unixTimeUTC := time.Unix(updateTimeStamp, 0)
+	// metada which we will PATCH deployment with
+	reqBody := body{
+		Metadata: &client.Metadata{
+			Annotations: map[string]string{
+				"build": unixTimeUTC.Format(time.RFC3339),
+			},
+		},
+	}
+	return k.client.UpdateDeployment(s.Name, reqBody)
+}
+
+// List the managed services
+func (k *kubernetes) List() ([]*runtime.Service, error) {
+	// TODO: this should list the k8s deployments
+	// but for now we return in-memory tracked services
+	var services []*runtime.Service
+	k.RLock()
+	defer k.RUnlock()
+
+	for _, service := range k.services {
+		services = append(services, service)
+	}
+
+	return services, nil
+}
+
+// run runs the runtime management loop
+func (k *kubernetes) run(events <-chan runtime.Event) {
+	t := time.NewTicker(time.Second * 5)
+	defer t.Stop()
+
+	for {
+		select {
+		case <-t.C:
+			// TODO: noop for now
+			// check running services
+			// * deployments exist
+			// * service is exposed
+		case service := <-k.start:
+			// TODO: following might have to be done
+			// * create a deployment
+			// * expose a service
+			log.Debugf("Runtime starting service: %s", service.Name)
+		case event := <-events:
+			// NOTE: we only handle Update events for now
+			log.Debugf("Runtime received notification event: %v", event)
+			switch event.Type {
+			case runtime.Update:
+				// parse returned response to timestamp
+				updateTimeStamp, err := strconv.ParseInt(event.Version, 10, 64)
+				if err != nil {
+					log.Debugf("Runtime error parsing update build time: %v", err)
+					continue
+				}
+				buildTime := time.Unix(updateTimeStamp, 0)
+				processEvent := func(event runtime.Event, service *runtime.Service) error {
+					buildTimeStamp, err := strconv.ParseInt(service.Version, 10, 64)
+					if err != nil {
+						return err
+					}
+					muBuild := time.Unix(buildTimeStamp, 0)
+					if buildTime.After(muBuild) {
+						version := fmt.Sprintf("%d", buildTime.Unix())
+						muService := &runtime.Service{
+							Name:    service.Name,
+							Source:  service.Source,
+							Path:    service.Path,
+							Exec:    service.Exec,
+							Version: version,
+						}
+						if err := k.Update(muService); err != nil {
+							return err
+						}
+						service.Version = version
+					}
+					return nil
+				}
+				k.Lock()
+				if len(event.Service) > 0 {
+					service, ok := k.services[event.Service]
+					if !ok {
+						log.Debugf("Runtime unknown service: %s", event.Service)
+						k.Unlock()
+						continue
+					}
+					if err := processEvent(event, service); err != nil {
+						log.Debugf("Runtime error updating service %s: %v", event.Service, err)
+					}
+					k.Unlock()
+					continue
+				}
+				// if blank service was received we update all services
+				for _, service := range k.services {
+					if err := processEvent(event, service); err != nil {
+						log.Debugf("Runtime error updating service %s: %v", service.Name, err)
+					}
+				}
+				k.Unlock()
+			}
+		case <-k.closed:
+			log.Debugf("Runtime stopped")
+			return
+		}
+	}
+}
+
+// starts the runtime
+func (k *kubernetes) Start() error {
+	k.Lock()
+	defer k.Unlock()
+
+	// already running
+	if k.running {
+		return nil
+	}
+
+	// set running
+	k.running = true
+	k.closed = make(chan bool)
+
+	var events <-chan runtime.Event
+	if k.options.Notifier != nil {
+		var err error
+		events, err = k.options.Notifier.Notify()
+		if err != nil {
+			// TODO: should we bail here?
+			log.Debugf("Runtime failed to start update notifier")
+		}
+	}
+
+	go k.run(events)
+
+	return nil
+}
+
+// Shutdown the runtime
+func (k *kubernetes) Stop() error {
+	k.Lock()
+	defer k.Unlock()
+
+	if !k.running {
+		return nil
+	}
+
+	select {
+	case <-k.closed:
+		return nil
+	default:
+		close(k.closed)
+		// set not running
+		k.running = false
+		// stop the notifier too
+		if k.options.Notifier != nil {
+			return k.options.Notifier.Close()
+		}
+	}
+
+	return nil
+}
+
+// String implements stringer interface
+func (k *kubernetes) String() string {
+	return "kubernetes"
+}

runtime/options.go

@@ -4,8 +4,24 @@ import (
 	"io"
 )
 
+type Option func(o *Options)
+
+// Options configure runtime
+type Options struct {
+	// Notifier for updates
+	Notifier Notifier
+}
+
+// AutoUpdate enables micro auto-updates
+func WithNotifier(n Notifier) Option {
+	return func(o *Options) {
+		o.Notifier = n
+	}
+}
+
 type CreateOption func(o *CreateOptions)
 
+// CreateOptions configure runtime services
 type CreateOptions struct {
 	// command to execute including args
 	Command []string
@@ -25,7 +41,7 @@ func WithCommand(c string, args ...string) CreateOption {
 	}
 }
 
-// WithEnv sets the created service env
+// WithEnv sets the created service environment
 func WithEnv(env []string) CreateOption {
 	return func(o *CreateOptions) {
 		o.Env = env

runtime/runtime.go

@@ -1,45 +1,87 @@
 // Package runtime is a service runtime manager
 package runtime
 
+import "time"
+
+var (
+	// DefaultRuntime is default micro runtime
+	DefaultRuntime Runtime = NewRuntime()
+)
+
 // Runtime is a service runtime manager
 type Runtime interface {
+	// Init initializes runtime
+	Init(...Option) error
 	// Registers a service
 	Create(*Service, ...CreateOption) error
 	// Remove a service
 	Delete(*Service) error
+	// Update the service in place
+	Update(*Service) error
+	// List the managed services
+	List() ([]*Service, error)
 	// starts the runtime
 	Start() error
 	// Shutdown the runtime
 	Stop() error
 }
 
+// Notifier is an update notifier
+type Notifier interface {
+	// Notify publishes notification events
+	Notify() (<-chan Event, error)
+	// Close stops the notifier
+	Close() error
+}
+
+// EventType defines notification event
+type EventType int
+
+const (
+	// Create is emitted when a new build has been craeted
+	Create EventType = iota
+	// Update is emitted when a new update become available
+	Update
+	// Delete is emitted when a build has been deleted
+	Delete
+)
+
+// String returns human readable event type
+func (t EventType) String() string {
+	switch t {
+	case Create:
+		return "create"
+	case Delete:
+		return "delete"
+	case Update:
+		return "update"
+	default:
+		return "unknown"
+	}
+}
+
+// Event is notification event
+type Event struct {
+	// Type is event type
+	Type EventType
+	// Timestamp is event timestamp
+	Timestamp time.Time
+	// Service is the name of the service
+	Service string
+	// Version of the build
+	Version string
+}
+
+// Service is runtime service
 type Service struct {
-	// name of the service
+	// Name of the service
 	Name string
 	// url location of source
 	Source string
-	// path to store source
+	// Path to store source
 	Path string
-	// exec command
+	// Exec command
 	Exec string
-}
-
-var (
-	DefaultRuntime = newRuntime()
-)
-
-func Create(s *Service, opts ...CreateOption) error {
-	return DefaultRuntime.Create(s, opts...)
-}
-
-func Delete(s *Service) error {
-	return DefaultRuntime.Delete(s)
-}
-
-func Start() error {
-	return DefaultRuntime.Start()
-}
-
-func Stop() error {
-	return DefaultRuntime.Stop()
+	// Version of the service
+	Version string
 }

runtime/service.go

@@ -0,0 +1,158 @@
+package runtime
+
+import (
+	"io"
+	"strings"
+	"sync"
+
+	packager "github.com/micro/go-micro/runtime/package"
+	"github.com/micro/go-micro/runtime/process"
+	proc "github.com/micro/go-micro/runtime/process/os"
+	"github.com/micro/go-micro/util/log"
+)
+
+type service struct {
+	sync.RWMutex
+
+	running bool
+	closed  chan bool
+	err     error
+
+	// output for logs
+	output io.Writer
+
+	// service to manage
+	*Service
+	// process creator
+	Process *proc.Process
+	// Exec
+	Exec *process.Executable
+	// process pid
+	PID *process.PID
+}
+
+func newService(s *Service, c CreateOptions) *service {
+	var exec string
+	var args []string
+
+	if len(s.Exec) > 0 {
+		parts := strings.Split(s.Exec, " ")
+		exec = parts[0]
+		args = []string{}
+
+		if len(parts) > 1 {
+			args = parts[1:]
+		}
+	} else {
+		// set command
+		exec = c.Command[0]
+		// set args
+		if len(c.Command) > 1 {
+			args = c.Command[1:]
+		}
+	}
+
+	return &service{
+		Service: s,
+		Process: new(proc.Process),
+		Exec: &process.Executable{
+			Binary: &packager.Binary{
+				Name: s.Name,
+				Path: exec,
+			},
+			Env:  c.Env,
+			Args: args,
+		},
+		closed: make(chan bool),
+		output: c.Output,
+	}
+}
+
+func (s *service) streamOutput() {
+	go io.Copy(s.output, s.PID.Output)
+	go io.Copy(s.output, s.PID.Error)
+}
+
+// Running returns true is the service is running
+func (s *service) Running() bool {
+	s.RLock()
+	defer s.RUnlock()
+	return s.running
+}
+
+// Start stars the service
+func (s *service) Start() error {
+	s.Lock()
+	defer s.Unlock()
+
+	if s.running {
+		return nil
+	}
+
+	// reset
+	s.err = nil
+	s.closed = make(chan bool)
+
+	// TODO: pull source & build binary
+	log.Debugf("Runtime service %s forking new process", s.Service.Name)
+	p, err := s.Process.Fork(s.Exec)
+	if err != nil {
+		return err
+	}
+
+	// set the pid
+	s.PID = p
+	// set to running
+	s.running = true
+
+	if s.output != nil {
+		s.streamOutput()
+	}
+
+	// wait and watch
+	go s.Wait()
+
+	return nil
+}
+
+// Stop stops the service
+func (s *service) Stop() error {
+	s.Lock()
+	defer s.Unlock()
+
+	select {
+	case <-s.closed:
+		return nil
+	default:
+		close(s.closed)
+		s.running = false
+		if s.PID == nil {
+			return nil
+		}
+		return s.Process.Kill(s.PID)
+	}
+}
+
+// Error returns the last error service has returned
+func (s *service) Error() error {
+	s.RLock()
+	defer s.RUnlock()
+	return s.err
+}
+
+// Wait waits for the service to finish running
+func (s *service) Wait() {
+	// wait for process to exit
+	err := s.Process.Wait(s.PID)
+
+	s.Lock()
+	defer s.Unlock()
+
+	// save the error
+	if err != nil {
+		s.err = err
+	}
+
+	// no longer running
+	s.running = false
+}

runtime/service/handler/handler.go

@@ -0,0 +1,91 @@
+package handler
+
+import (
+	"context"
+
+	"github.com/micro/go-micro/errors"
+	"github.com/micro/go-micro/runtime"
+	pb "github.com/micro/go-micro/runtime/service/proto"
+)
+
+type Runtime struct {
+	Runtime runtime.Runtime
+}
+
+func toProto(s *runtime.Service) *pb.Service {
+	return &pb.Service{
+		Name:    s.Name,
+		Version: s.Version,
+		Source:  s.Source,
+		Path:    s.Path,
+		Exec:    s.Exec,
+	}
+}
+
+func toService(s *pb.Service) *runtime.Service {
+	return &runtime.Service{
+		Name:    s.Name,
+		Version: s.Version,
+		Source:  s.Source,
+		Path:    s.Path,
+		Exec:    s.Exec,
+	}
+}
+
+func (r *Runtime) Create(ctx context.Context, req *pb.CreateRequest, rsp *pb.CreateResponse) error {
+	if req.Service == nil {
+		return errors.BadRequest("go.micro.runtime", "blank service")
+	}
+
+	// TODO: add opts
+	service := toService(req.Service)
+	err := r.Runtime.Create(service)
+	if err != nil {
+		return errors.InternalServerError("go.micro.runtime", err.Error())
+	}
+
+	return nil
+}
+
+func (r *Runtime) Update(ctx context.Context, req *pb.UpdateRequest, rsp *pb.UpdateResponse) error {
+	if req.Service == nil {
+		return errors.BadRequest("go.micro.runtime", "blank service")
+	}
+
+	// TODO: add opts
+	service := toService(req.Service)
+	err := r.Runtime.Update(service)
+	if err != nil {
+		return errors.InternalServerError("go.micro.runtime", err.Error())
+	}
+
+	return nil
+}
+
+func (r *Runtime) Delete(ctx context.Context, req *pb.DeleteRequest, rsp *pb.DeleteResponse) error {
+	if req.Service == nil {
+		return errors.BadRequest("go.micro.runtime", "blank service")
+	}
+
+	// TODO: add opts
+	service := toService(req.Service)
+	err := r.Runtime.Delete(service)
+	if err != nil {
+		return errors.InternalServerError("go.micro.runtime", err.Error())
+	}
+
+	return nil
+}
+
+func (r *Runtime) List(ctx context.Context, req *pb.ListRequest, rsp *pb.ListResponse) error {
+	services, err := r.Runtime.List()
+	if err != nil {
+		return errors.InternalServerError("go.micro.runtime", err.Error())
+	}
+
+	for _, service := range services {
+		rsp.Services = append(rsp.Services, toProto(service))
+	}
+
+	return nil
+}

runtime/service/proto/runtime.micro.go

@@ -0,0 +1,142 @@
+// Code generated by protoc-gen-micro. DO NOT EDIT.
+// source: micro/go-micro/runtime/service/proto/runtime.proto
+
+package go_micro_runtime
+
+import (
+	fmt "fmt"
+	proto "github.com/golang/protobuf/proto"
+	math "math"
+)
+
+import (
+	context "context"
+	client "github.com/micro/go-micro/client"
+	server "github.com/micro/go-micro/server"
+)
+
+// Reference imports to suppress errors if they are not otherwise used.
+var _ = proto.Marshal
+var _ = fmt.Errorf
+var _ = math.Inf
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the proto package it is being compiled against.
+// A compilation error at this line likely means your copy of the
+// proto package needs to be updated.
+const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package
+
+// Reference imports to suppress errors if they are not otherwise used.
+var _ context.Context
+var _ client.Option
+var _ server.Option
+
+// Client API for Runtime service
+
+type RuntimeService interface {
+	Create(ctx context.Context, in *CreateRequest, opts ...client.CallOption) (*CreateResponse, error)
+	Delete(ctx context.Context, in *DeleteRequest, opts ...client.CallOption) (*DeleteResponse, error)
+	Update(ctx context.Context, in *UpdateRequest, opts ...client.CallOption) (*UpdateResponse, error)
+	List(ctx context.Context, in *ListRequest, opts ...client.CallOption) (*ListResponse, error)
+}
+
+type runtimeService struct {
+	c    client.Client
+	name string
+}
+
+func NewRuntimeService(name string, c client.Client) RuntimeService {
+	if c == nil {
+		c = client.NewClient()
+	}
+	if len(name) == 0 {
+		name = "go.micro.runtime"
+	}
+	return &runtimeService{
+		c:    c,
+		name: name,
+	}
+}
+
+func (c *runtimeService) Create(ctx context.Context, in *CreateRequest, opts ...client.CallOption) (*CreateResponse, error) {
+	req := c.c.NewRequest(c.name, "Runtime.Create", in)
+	out := new(CreateResponse)
+	err := c.c.Call(ctx, req, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *runtimeService) Delete(ctx context.Context, in *DeleteRequest, opts ...client.CallOption) (*DeleteResponse, error) {
+	req := c.c.NewRequest(c.name, "Runtime.Delete", in)
+	out := new(DeleteResponse)
+	err := c.c.Call(ctx, req, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *runtimeService) Update(ctx context.Context, in *UpdateRequest, opts ...client.CallOption) (*UpdateResponse, error) {
+	req := c.c.NewRequest(c.name, "Runtime.Update", in)
+	out := new(UpdateResponse)
+	err := c.c.Call(ctx, req, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *runtimeService) List(ctx context.Context, in *ListRequest, opts ...client.CallOption) (*ListResponse, error) {
+	req := c.c.NewRequest(c.name, "Runtime.List", in)
+	out := new(ListResponse)
+	err := c.c.Call(ctx, req, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+// Server API for Runtime service
+
+type RuntimeHandler interface {
+	Create(context.Context, *CreateRequest, *CreateResponse) error
+	Delete(context.Context, *DeleteRequest, *DeleteResponse) error
+	Update(context.Context, *UpdateRequest, *UpdateResponse) error
+	List(context.Context, *ListRequest, *ListResponse) error
+}
+
+func RegisterRuntimeHandler(s server.Server, hdlr RuntimeHandler, opts ...server.HandlerOption) error {
+	type runtime interface {
+		Create(ctx context.Context, in *CreateRequest, out *CreateResponse) error
+		Delete(ctx context.Context, in *DeleteRequest, out *DeleteResponse) error
+		Update(ctx context.Context, in *UpdateRequest, out *UpdateResponse) error
+		List(ctx context.Context, in *ListRequest, out *ListResponse) error
+	}
+	type Runtime struct {
+		runtime
+	}
+	h := &runtimeHandler{hdlr}
+	return s.Handle(s.NewHandler(&Runtime{h}, opts...))
+}
+
+type runtimeHandler struct {
+	RuntimeHandler
+}
+
+func (h *runtimeHandler) Create(ctx context.Context, in *CreateRequest, out *CreateResponse) error {
+	return h.RuntimeHandler.Create(ctx, in, out)
+}
+
+func (h *runtimeHandler) Delete(ctx context.Context, in *DeleteRequest, out *DeleteResponse) error {
+	return h.RuntimeHandler.Delete(ctx, in, out)
+}
+
+func (h *runtimeHandler) Update(ctx context.Context, in *UpdateRequest, out *UpdateResponse) error {
+	return h.RuntimeHandler.Update(ctx, in, out)
+}
+
+func (h *runtimeHandler) List(ctx context.Context, in *ListRequest, out *ListResponse) error {
+	return h.RuntimeHandler.List(ctx, in, out)
+}

runtime/service/proto/runtime.pb.go

@@ -0,0 +1,661 @@
+// Code generated by protoc-gen-go. DO NOT EDIT.
+// source: micro/go-micro/runtime/service/proto/runtime.proto
+
+package go_micro_runtime
+
+import (
+	context "context"
+	fmt "fmt"
+	proto "github.com/golang/protobuf/proto"
+	grpc "google.golang.org/grpc"
+	math "math"
+)
+
+// Reference imports to suppress errors if they are not otherwise used.
+var _ = proto.Marshal
+var _ = fmt.Errorf
+var _ = math.Inf
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the proto package it is being compiled against.
+// A compilation error at this line likely means your copy of the
+// proto package needs to be updated.
+const _ = proto.ProtoPackageIsVersion3 // please upgrade the proto package
+
+type Service struct {
+	// name of the service
+	Name string `protobuf:"bytes,1,opt,name=name,proto3" json:"name,omitempty"`
+	// version of the service
+	Version string `protobuf:"bytes,2,opt,name=version,proto3" json:"version,omitempty"`
+	// git url of the source
+	Source string `protobuf:"bytes,3,opt,name=source,proto3" json:"source,omitempty"`
+	// local path of the source
+	Path string `protobuf:"bytes,4,opt,name=path,proto3" json:"path,omitempty"`
+	// command to execute
+	Exec                 string   `protobuf:"bytes,5,opt,name=exec,proto3" json:"exec,omitempty"`
+	XXX_NoUnkeyedLiteral struct{} `json:"-"`
+	XXX_unrecognized     []byte   `json:"-"`
+	XXX_sizecache        int32    `json:"-"`
+}
+
+func (m *Service) Reset()         { *m = Service{} }
+func (m *Service) String() string { return proto.CompactTextString(m) }
+func (*Service) ProtoMessage()    {}
+func (*Service) Descriptor() ([]byte, []int) {
+	return fileDescriptor_4bc91a8efec81434, []int{0}
+}
+
+func (m *Service) XXX_Unmarshal(b []byte) error {
+	return xxx_messageInfo_Service.Unmarshal(m, b)
+}
+func (m *Service) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	return xxx_messageInfo_Service.Marshal(b, m, deterministic)
+}
+func (m *Service) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_Service.Merge(m, src)
+}
+func (m *Service) XXX_Size() int {
+	return xxx_messageInfo_Service.Size(m)
+}
+func (m *Service) XXX_DiscardUnknown() {
+	xxx_messageInfo_Service.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_Service proto.InternalMessageInfo
+
+func (m *Service) GetName() string {
+	if m != nil {
+		return m.Name
+	}
+	return ""
+}
+
+func (m *Service) GetVersion() string {
+	if m != nil {
+		return m.Version
+	}
+	return ""
+}
+
+func (m *Service) GetSource() string {
+	if m != nil {
+		return m.Source
+	}
+	return ""
+}
+
+func (m *Service) GetPath() string {
+	if m != nil {
+		return m.Path
+	}
+	return ""
+}
+
+func (m *Service) GetExec() string {
+	if m != nil {
+		return m.Exec
+	}
+	return ""
+}
+
+type Options struct {
+	// command to pass in
+	Command string `protobuf:"bytes,1,opt,name=command,proto3" json:"command,omitempty"`
+	// environment to pass in
+	Env []string `protobuf:"bytes,2,rep,name=env,proto3" json:"env,omitempty"`
+	// output to send to
+	Output               string   `protobuf:"bytes,3,opt,name=output,proto3" json:"output,omitempty"`
+	XXX_NoUnkeyedLiteral struct{} `json:"-"`
+	XXX_unrecognized     []byte   `json:"-"`
+	XXX_sizecache        int32    `json:"-"`
+}
+
+func (m *Options) Reset()         { *m = Options{} }
+func (m *Options) String() string { return proto.CompactTextString(m) }
+func (*Options) ProtoMessage()    {}
+func (*Options) Descriptor() ([]byte, []int) {
+	return fileDescriptor_4bc91a8efec81434, []int{1}
+}
+
+func (m *Options) XXX_Unmarshal(b []byte) error {
+	return xxx_messageInfo_Options.Unmarshal(m, b)
+}
+func (m *Options) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	return xxx_messageInfo_Options.Marshal(b, m, deterministic)
+}
+func (m *Options) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_Options.Merge(m, src)
+}
+func (m *Options) XXX_Size() int {
+	return xxx_messageInfo_Options.Size(m)
+}
+func (m *Options) XXX_DiscardUnknown() {
+	xxx_messageInfo_Options.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_Options proto.InternalMessageInfo
+
+func (m *Options) GetCommand() string {
+	if m != nil {
+		return m.Command
+	}
+	return ""
+}
+
+func (m *Options) GetEnv() []string {
+	if m != nil {
+		return m.Env
+	}
+	return nil
+}
+
+func (m *Options) GetOutput() string {
+	if m != nil {
+		return m.Output
+	}
+	return ""
+}
+
+type CreateRequest struct {
+	Service              *Service `protobuf:"bytes,1,opt,name=service,proto3" json:"service,omitempty"`
+	Options              *Options `protobuf:"bytes,2,opt,name=options,proto3" json:"options,omitempty"`
+	XXX_NoUnkeyedLiteral struct{} `json:"-"`
+	XXX_unrecognized     []byte   `json:"-"`
+	XXX_sizecache        int32    `json:"-"`
+}
+
+func (m *CreateRequest) Reset()         { *m = CreateRequest{} }
+func (m *CreateRequest) String() string { return proto.CompactTextString(m) }
+func (*CreateRequest) ProtoMessage()    {}
+func (*CreateRequest) Descriptor() ([]byte, []int) {
+	return fileDescriptor_4bc91a8efec81434, []int{2}
+}
+
+func (m *CreateRequest) XXX_Unmarshal(b []byte) error {
+	return xxx_messageInfo_CreateRequest.Unmarshal(m, b)
+}
+func (m *CreateRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	return xxx_messageInfo_CreateRequest.Marshal(b, m, deterministic)
+}
+func (m *CreateRequest) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_CreateRequest.Merge(m, src)
+}
+func (m *CreateRequest) XXX_Size() int {
+	return xxx_messageInfo_CreateRequest.Size(m)
+}
+func (m *CreateRequest) XXX_DiscardUnknown() {
+	xxx_messageInfo_CreateRequest.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_CreateRequest proto.InternalMessageInfo
+
+func (m *CreateRequest) GetService() *Service {
+	if m != nil {
+		return m.Service
+	}
+	return nil
+}
+
+func (m *CreateRequest) GetOptions() *Options {
+	if m != nil {
+		return m.Options
+	}
+	return nil
+}
+
+type CreateResponse struct {
+	XXX_NoUnkeyedLiteral struct{} `json:"-"`
+	XXX_unrecognized     []byte   `json:"-"`
+	XXX_sizecache        int32    `json:"-"`
+}
+
+func (m *CreateResponse) Reset()         { *m = CreateResponse{} }
+func (m *CreateResponse) String() string { return proto.CompactTextString(m) }
+func (*CreateResponse) ProtoMessage()    {}
+func (*CreateResponse) Descriptor() ([]byte, []int) {
+	return fileDescriptor_4bc91a8efec81434, []int{3}
+}
+
+func (m *CreateResponse) XXX_Unmarshal(b []byte) error {
+	return xxx_messageInfo_CreateResponse.Unmarshal(m, b)
+}
+func (m *CreateResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	return xxx_messageInfo_CreateResponse.Marshal(b, m, deterministic)
+}
+func (m *CreateResponse) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_CreateResponse.Merge(m, src)
+}
+func (m *CreateResponse) XXX_Size() int {
+	return xxx_messageInfo_CreateResponse.Size(m)
+}
+func (m *CreateResponse) XXX_DiscardUnknown() {
+	xxx_messageInfo_CreateResponse.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_CreateResponse proto.InternalMessageInfo
+
+type DeleteRequest struct {
+	Service              *Service `protobuf:"bytes,1,opt,name=service,proto3" json:"service,omitempty"`
+	XXX_NoUnkeyedLiteral struct{} `json:"-"`
+	XXX_unrecognized     []byte   `json:"-"`
+	XXX_sizecache        int32    `json:"-"`
+}
+
+func (m *DeleteRequest) Reset()         { *m = DeleteRequest{} }
+func (m *DeleteRequest) String() string { return proto.CompactTextString(m) }
+func (*DeleteRequest) ProtoMessage()    {}
+func (*DeleteRequest) Descriptor() ([]byte, []int) {
+	return fileDescriptor_4bc91a8efec81434, []int{4}
+}
+
+func (m *DeleteRequest) XXX_Unmarshal(b []byte) error {
+	return xxx_messageInfo_DeleteRequest.Unmarshal(m, b)
+}
+func (m *DeleteRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	return xxx_messageInfo_DeleteRequest.Marshal(b, m, deterministic)
+}
+func (m *DeleteRequest) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_DeleteRequest.Merge(m, src)
+}
+func (m *DeleteRequest) XXX_Size() int {
+	return xxx_messageInfo_DeleteRequest.Size(m)
+}
+func (m *DeleteRequest) XXX_DiscardUnknown() {
+	xxx_messageInfo_DeleteRequest.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_DeleteRequest proto.InternalMessageInfo
+
+func (m *DeleteRequest) GetService() *Service {
+	if m != nil {
+		return m.Service
+	}
+	return nil
+}
+
+type DeleteResponse struct {
+	XXX_NoUnkeyedLiteral struct{} `json:"-"`
+	XXX_unrecognized     []byte   `json:"-"`
+	XXX_sizecache        int32    `json:"-"`
+}
+
+func (m *DeleteResponse) Reset()         { *m = DeleteResponse{} }
+func (m *DeleteResponse) String() string { return proto.CompactTextString(m) }
+func (*DeleteResponse) ProtoMessage()    {}
+func (*DeleteResponse) Descriptor() ([]byte, []int) {
+	return fileDescriptor_4bc91a8efec81434, []int{5}
+}
+
+func (m *DeleteResponse) XXX_Unmarshal(b []byte) error {
+	return xxx_messageInfo_DeleteResponse.Unmarshal(m, b)
+}
+func (m *DeleteResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	return xxx_messageInfo_DeleteResponse.Marshal(b, m, deterministic)
+}
+func (m *DeleteResponse) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_DeleteResponse.Merge(m, src)
+}
+func (m *DeleteResponse) XXX_Size() int {
+	return xxx_messageInfo_DeleteResponse.Size(m)
+}
+func (m *DeleteResponse) XXX_DiscardUnknown() {
+	xxx_messageInfo_DeleteResponse.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_DeleteResponse proto.InternalMessageInfo
+
+type UpdateRequest struct {
+	Service              *Service `protobuf:"bytes,1,opt,name=service,proto3" json:"service,omitempty"`
+	XXX_NoUnkeyedLiteral struct{} `json:"-"`
+	XXX_unrecognized     []byte   `json:"-"`
+	XXX_sizecache        int32    `json:"-"`
+}
+
+func (m *UpdateRequest) Reset()         { *m = UpdateRequest{} }
+func (m *UpdateRequest) String() string { return proto.CompactTextString(m) }
+func (*UpdateRequest) ProtoMessage()    {}
+func (*UpdateRequest) Descriptor() ([]byte, []int) {
+	return fileDescriptor_4bc91a8efec81434, []int{6}
+}
+
+func (m *UpdateRequest) XXX_Unmarshal(b []byte) error {
+	return xxx_messageInfo_UpdateRequest.Unmarshal(m, b)
+}
+func (m *UpdateRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	return xxx_messageInfo_UpdateRequest.Marshal(b, m, deterministic)
+}
+func (m *UpdateRequest) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_UpdateRequest.Merge(m, src)
+}
+func (m *UpdateRequest) XXX_Size() int {
+	return xxx_messageInfo_UpdateRequest.Size(m)
+}
+func (m *UpdateRequest) XXX_DiscardUnknown() {
+	xxx_messageInfo_UpdateRequest.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_UpdateRequest proto.InternalMessageInfo
+
+func (m *UpdateRequest) GetService() *Service {
+	if m != nil {
+		return m.Service
+	}
+	return nil
+}
+
+type UpdateResponse struct {
+	XXX_NoUnkeyedLiteral struct{} `json:"-"`
+	XXX_unrecognized     []byte   `json:"-"`
+	XXX_sizecache        int32    `json:"-"`
+}
+
+func (m *UpdateResponse) Reset()         { *m = UpdateResponse{} }
+func (m *UpdateResponse) String() string { return proto.CompactTextString(m) }
+func (*UpdateResponse) ProtoMessage()    {}
+func (*UpdateResponse) Descriptor() ([]byte, []int) {
+	return fileDescriptor_4bc91a8efec81434, []int{7}
+}
+
+func (m *UpdateResponse) XXX_Unmarshal(b []byte) error {
+	return xxx_messageInfo_UpdateResponse.Unmarshal(m, b)
+}
+func (m *UpdateResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	return xxx_messageInfo_UpdateResponse.Marshal(b, m, deterministic)
+}
+func (m *UpdateResponse) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_UpdateResponse.Merge(m, src)
+}
+func (m *UpdateResponse) XXX_Size() int {
+	return xxx_messageInfo_UpdateResponse.Size(m)
+}
+func (m *UpdateResponse) XXX_DiscardUnknown() {
+	xxx_messageInfo_UpdateResponse.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_UpdateResponse proto.InternalMessageInfo
+
+type ListRequest struct {
+	XXX_NoUnkeyedLiteral struct{} `json:"-"`
+	XXX_unrecognized     []byte   `json:"-"`
+	XXX_sizecache        int32    `json:"-"`
+}
+
+func (m *ListRequest) Reset()         { *m = ListRequest{} }
+func (m *ListRequest) String() string { return proto.CompactTextString(m) }
+func (*ListRequest) ProtoMessage()    {}
+func (*ListRequest) Descriptor() ([]byte, []int) {
+	return fileDescriptor_4bc91a8efec81434, []int{8}
+}
+
+func (m *ListRequest) XXX_Unmarshal(b []byte) error {
+	return xxx_messageInfo_ListRequest.Unmarshal(m, b)
+}
+func (m *ListRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	return xxx_messageInfo_ListRequest.Marshal(b, m, deterministic)
+}
+func (m *ListRequest) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_ListRequest.Merge(m, src)
+}
+func (m *ListRequest) XXX_Size() int {
+	return xxx_messageInfo_ListRequest.Size(m)
+}
+func (m *ListRequest) XXX_DiscardUnknown() {
+	xxx_messageInfo_ListRequest.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_ListRequest proto.InternalMessageInfo
+
+type ListResponse struct {
+	Services             []*Service `protobuf:"bytes,1,rep,name=services,proto3" json:"services,omitempty"`
+	XXX_NoUnkeyedLiteral struct{}   `json:"-"`
+	XXX_unrecognized     []byte     `json:"-"`
+	XXX_sizecache        int32      `json:"-"`
+}
+
+func (m *ListResponse) Reset()         { *m = ListResponse{} }
+func (m *ListResponse) String() string { return proto.CompactTextString(m) }
+func (*ListResponse) ProtoMessage()    {}
+func (*ListResponse) Descriptor() ([]byte, []int) {
+	return fileDescriptor_4bc91a8efec81434, []int{9}
+}
+
+func (m *ListResponse) XXX_Unmarshal(b []byte) error {
+	return xxx_messageInfo_ListResponse.Unmarshal(m, b)
+}
+func (m *ListResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	return xxx_messageInfo_ListResponse.Marshal(b, m, deterministic)
+}
+func (m *ListResponse) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_ListResponse.Merge(m, src)
+}
+func (m *ListResponse) XXX_Size() int {
+	return xxx_messageInfo_ListResponse.Size(m)
+}
+func (m *ListResponse) XXX_DiscardUnknown() {
+	xxx_messageInfo_ListResponse.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_ListResponse proto.InternalMessageInfo
+
+func (m *ListResponse) GetServices() []*Service {
+	if m != nil {
+		return m.Services
+	}
+	return nil
+}
+
+func init() {
+	proto.RegisterType((*Service)(nil), "go.micro.runtime.Service")
+	proto.RegisterType((*Options)(nil), "go.micro.runtime.Options")
+	proto.RegisterType((*CreateRequest)(nil), "go.micro.runtime.CreateRequest")
+	proto.RegisterType((*CreateResponse)(nil), "go.micro.runtime.CreateResponse")
+	proto.RegisterType((*DeleteRequest)(nil), "go.micro.runtime.DeleteRequest")
+	proto.RegisterType((*DeleteResponse)(nil), "go.micro.runtime.DeleteResponse")
+	proto.RegisterType((*UpdateRequest)(nil), "go.micro.runtime.UpdateRequest")
+	proto.RegisterType((*UpdateResponse)(nil), "go.micro.runtime.UpdateResponse")
+	proto.RegisterType((*ListRequest)(nil), "go.micro.runtime.ListRequest")
+	proto.RegisterType((*ListResponse)(nil), "go.micro.runtime.ListResponse")
+}
+
+func init() {
+	proto.RegisterFile("micro/go-micro/runtime/service/proto/runtime.proto", fileDescriptor_4bc91a8efec81434)
+}
+
+var fileDescriptor_4bc91a8efec81434 = []byte{
+	// 379 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xa4, 0x93, 0xcb, 0x4e, 0xc2, 0x40,
+	0x14, 0x86, 0x81, 0x22, 0x95, 0x83, 0x18, 0x32, 0x0b, 0x33, 0x92, 0xa8, 0x4d, 0x57, 0x6c, 0x2c,
+	0x09, 0xc4, 0x27, 0x10, 0xe3, 0x46, 0x62, 0x52, 0xe3, 0x03, 0xd4, 0x72, 0x82, 0x4d, 0x6c, 0xa7,
+	0xce, 0x4c, 0x51, 0x9f, 0xce, 0x57, 0x33, 0x73, 0x43, 0x81, 0xe2, 0x86, 0xdd, 0xb9, 0x7e, 0xff,
+	0x99, 0xbf, 0x29, 0x4c, 0xf2, 0x2c, 0xe5, 0x6c, 0xbc, 0x64, 0xd7, 0x26, 0xe0, 0x55, 0x21, 0xb3,
+	0x1c, 0xc7, 0x02, 0xf9, 0x2a, 0x4b, 0x71, 0x5c, 0x72, 0x26, 0xd7, 0xd5, 0x48, 0x67, 0x64, 0xb0,
+	0x64, 0x91, 0x9e, 0x8e, 0x6c, 0x3d, 0xfc, 0x00, 0xff, 0xc9, 0x2c, 0x10, 0x02, 0xed, 0x22, 0xc9,
+	0x91, 0x36, 0x83, 0xe6, 0xa8, 0x1b, 0xeb, 0x98, 0x50, 0xf0, 0x57, 0xc8, 0x45, 0xc6, 0x0a, 0xda,
+	0xd2, 0x65, 0x97, 0x92, 0x33, 0xe8, 0x08, 0x56, 0xf1, 0x14, 0xa9, 0xa7, 0x1b, 0x36, 0x53, 0x94,
+	0x32, 0x91, 0xaf, 0xb4, 0x6d, 0x28, 0x2a, 0x56, 0x35, 0xfc, 0xc4, 0x94, 0x1e, 0x99, 0x9a, 0x8a,
+	0xc3, 0x39, 0xf8, 0x8f, 0xa5, 0xcc, 0x58, 0x21, 0x94, 0x48, 0xca, 0xf2, 0x3c, 0x29, 0x16, 0x56,
+	0xdb, 0xa5, 0x64, 0x00, 0x1e, 0x16, 0x2b, 0xda, 0x0a, 0xbc, 0x51, 0x37, 0x56, 0xa1, 0x92, 0x65,
+	0x95, 0x2c, 0x2b, 0xe9, 0x64, 0x4d, 0x16, 0x7e, 0x41, 0xff, 0x96, 0x63, 0x22, 0x31, 0xc6, 0xf7,
+	0x0a, 0x85, 0x24, 0x53, 0xf0, 0xad, 0x13, 0x1a, 0xda, 0x9b, 0x9c, 0x47, 0xdb, 0x8f, 0x8f, 0xec,
+	0xcb, 0x63, 0x37, 0xa9, 0x96, 0x98, 0x39, 0x4a, 0x3f, 0xb7, 0x76, 0xc9, 0x5e, 0x1d, 0xbb, 0xc9,
+	0x70, 0x00, 0xa7, 0x4e, 0x5a, 0x94, 0xac, 0x10, 0x18, 0xce, 0xa0, 0x3f, 0xc3, 0x37, 0x3c, 0xec,
+	0x18, 0xc5, 0x75, 0x94, 0x5f, 0xee, 0x73, 0xb9, 0x48, 0x0e, 0xe7, 0x3a, 0x8a, 0xe5, 0xf6, 0xa1,
+	0xf7, 0x90, 0x09, 0x69, 0xa9, 0xe1, 0x1d, 0x9c, 0x98, 0xd4, 0xb4, 0xc9, 0x0d, 0x1c, 0xdb, 0x5d,
+	0x41, 0x9b, 0x81, 0xf7, 0xbf, 0xcc, 0x7a, 0x74, 0xf2, 0xdd, 0x02, 0x3f, 0x36, 0x5d, 0x32, 0x87,
+	0x8e, 0xf1, 0x88, 0x5c, 0xed, 0xae, 0x6e, 0x7c, 0xb8, 0x61, 0xb0, 0x7f, 0xc0, 0x9e, 0xdb, 0x50,
+	0x38, 0x63, 0x4d, 0x1d, 0x6e, 0xc3, 0xfa, 0x3a, 0xdc, 0x96, 0xab, 0x1a, 0x67, 0x1c, 0xa9, 0xc3,
+	0x6d, 0x38, 0x5e, 0x87, 0xdb, 0x32, 0xb3, 0x41, 0xee, 0xa1, 0xad, 0xfc, 0x23, 0x17, 0xbb, 0xb3,
+	0x7f, 0x6c, 0x1e, 0x5e, 0xee, 0x6b, 0x3b, 0xd0, 0x4b, 0x47, 0xff, 0xb5, 0xd3, 0x9f, 0x00, 0x00,
+	0x00, 0xff, 0xff, 0x69, 0x49, 0x0f, 0xe1, 0xeb, 0x03, 0x00, 0x00,
+}
+
+// Reference imports to suppress errors if they are not otherwise used.
+var _ context.Context
+var _ grpc.ClientConn
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the grpc package it is being compiled against.
+const _ = grpc.SupportPackageIsVersion4
+
+// RuntimeClient is the client API for Runtime service.
+//
+// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream.
+type RuntimeClient interface {
+	Create(ctx context.Context, in *CreateRequest, opts ...grpc.CallOption) (*CreateResponse, error)
+	Delete(ctx context.Context, in *DeleteRequest, opts ...grpc.CallOption) (*DeleteResponse, error)
+	Update(ctx context.Context, in *UpdateRequest, opts ...grpc.CallOption) (*UpdateResponse, error)
+	List(ctx context.Context, in *ListRequest, opts ...grpc.CallOption) (*ListResponse, error)
+}
+
+type runtimeClient struct {
+	cc *grpc.ClientConn
+}
+
+func NewRuntimeClient(cc *grpc.ClientConn) RuntimeClient {
+	return &runtimeClient{cc}
+}
+
+func (c *runtimeClient) Create(ctx context.Context, in *CreateRequest, opts ...grpc.CallOption) (*CreateResponse, error) {
+	out := new(CreateResponse)
+	err := c.cc.Invoke(ctx, "/go.micro.runtime.Runtime/Create", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *runtimeClient) Delete(ctx context.Context, in *DeleteRequest, opts ...grpc.CallOption) (*DeleteResponse, error) {
+	out := new(DeleteResponse)
+	err := c.cc.Invoke(ctx, "/go.micro.runtime.Runtime/Delete", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *runtimeClient) Update(ctx context.Context, in *UpdateRequest, opts ...grpc.CallOption) (*UpdateResponse, error) {
+	out := new(UpdateResponse)
+	err := c.cc.Invoke(ctx, "/go.micro.runtime.Runtime/Update", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *runtimeClient) List(ctx context.Context, in *ListRequest, opts ...grpc.CallOption) (*ListResponse, error) {
+	out := new(ListResponse)
+	err := c.cc.Invoke(ctx, "/go.micro.runtime.Runtime/List", in, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+// RuntimeServer is the server API for Runtime service.
+type RuntimeServer interface {
+	Create(context.Context, *CreateRequest) (*CreateResponse, error)
+	Delete(context.Context, *DeleteRequest) (*DeleteResponse, error)
+	Update(context.Context, *UpdateRequest) (*UpdateResponse, error)
+	List(context.Context, *ListRequest) (*ListResponse, error)
+}
+
+func RegisterRuntimeServer(s *grpc.Server, srv RuntimeServer) {
+	s.RegisterService(&_Runtime_serviceDesc, srv)
+}
+
+func _Runtime_Create_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(CreateRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(RuntimeServer).Create(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/go.micro.runtime.Runtime/Create",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(RuntimeServer).Create(ctx, req.(*CreateRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _Runtime_Delete_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(DeleteRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(RuntimeServer).Delete(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/go.micro.runtime.Runtime/Delete",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(RuntimeServer).Delete(ctx, req.(*DeleteRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _Runtime_Update_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(UpdateRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(RuntimeServer).Update(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/go.micro.runtime.Runtime/Update",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(RuntimeServer).Update(ctx, req.(*UpdateRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+func _Runtime_List_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
+	in := new(ListRequest)
+	if err := dec(in); err != nil {
+		return nil, err
+	}
+	if interceptor == nil {
+		return srv.(RuntimeServer).List(ctx, in)
+	}
+	info := &grpc.UnaryServerInfo{
+		Server:     srv,
+		FullMethod: "/go.micro.runtime.Runtime/List",
+	}
+	handler := func(ctx context.Context, req interface{}) (interface{}, error) {
+		return srv.(RuntimeServer).List(ctx, req.(*ListRequest))
+	}
+	return interceptor(ctx, in, info, handler)
+}
+
+var _Runtime_serviceDesc = grpc.ServiceDesc{
+	ServiceName: "go.micro.runtime.Runtime",
+	HandlerType: (*RuntimeServer)(nil),
+	Methods: []grpc.MethodDesc{
+		{
+			MethodName: "Create",
+			Handler:    _Runtime_Create_Handler,
+		},
+		{
+			MethodName: "Delete",
+			Handler:    _Runtime_Delete_Handler,
+		},
+		{
+			MethodName: "Update",
+			Handler:    _Runtime_Update_Handler,
+		},
+		{
+			MethodName: "List",
+			Handler:    _Runtime_List_Handler,
+		},
+	},
+	Streams:  []grpc.StreamDesc{},
+	Metadata: "micro/go-micro/runtime/service/proto/runtime.proto",
+}

runtime/service/proto/runtime.proto

@@ -0,0 +1,57 @@
+syntax = "proto3";
+
+package go.micro.runtime;
+
+service Runtime {
+	rpc Create(CreateRequest) returns (CreateResponse) {};
+	rpc Delete(DeleteRequest) returns (DeleteResponse) {};
+	rpc Update(UpdateRequest) returns (UpdateResponse) {};
+	rpc List(ListRequest) returns (ListResponse) {};
+}
+
+message Service {
+	// name of the service
+	string name = 1;
+	// version of the service
+	string version = 2;
+	// git url of the source
+	string source = 3;
+	// local path of the source
+	string path = 4;
+	// command to execute
+	string exec = 5;
+}
+
+message Options {
+	// command to pass in
+	string command = 1;
+	// environment to pass in
+	repeated string env = 2;
+	// output to send to
+	string output = 3;
+}
+
+message CreateRequest {
+	Service service = 1;
+	Options options = 2;
+}
+
+message CreateResponse {}
+
+message DeleteRequest {
+	Service service = 1;
+}
+
+message DeleteResponse {}
+
+message UpdateRequest {
+	Service service = 1;
+}
+
+message UpdateResponse {}
+
+message ListRequest {}
+
+message ListResponse {
+	repeated Service services = 1;
+}

server/grpc/grpc.go

@@ -7,6 +7,7 @@ import (
 	"fmt"
 	"net"
 	"reflect"
+	"runtime/debug"
 	"sort"
 	"strconv"
 	"strings"
@@ -340,7 +341,8 @@ func (g *grpcServer) processRequest(stream grpc.ServerStream, service *service,
 		fn := func(ctx context.Context, req server.Request, rsp interface{}) error {
 			defer func() {
 				if r := recover(); r != nil {
-					log.Logf("handler %s panic recovered, err: %s", mtype.method.Name, r)
+					log.Log("panic recovered: ", r)
+					log.Logf(string(debug.Stack()))
 				}
 			}()
 			returnValues = function.Call([]reflect.Value{service.rcvr, mtype.prepareContext(ctx), reflect.ValueOf(argv.Interface()), reflect.ValueOf(rsp)})
@@ -621,7 +623,7 @@ func (g *grpcServer) Register() error {
 
 	g.registered = true
 
-	for sb, _ := range g.subscribers {
+	for sb := range g.subscribers {
 		handler := g.createSubHandler(sb, g.opts)
 		var opts []broker.SubscribeOption
 		if queue := sb.Options().Queue; len(queue) > 0 {

server/rpc_server.go

@@ -531,7 +531,7 @@ func (s *rpcServer) Register() error {
 
 	s.registered = true
 
-	for sb, _ := range s.subscribers {
+	for sb := range s.subscribers {
 		handler := s.createSubHandler(sb, s.opts)
 		var opts []broker.SubscribeOption
 		if queue := sb.Options().Queue; len(queue) > 0 {

store/cloudflare/cloudflare.go

@@ -3,76 +3,132 @@
 package cloudflare
 
 import (
+	"bytes"
 	"context"
+	"encoding/json"
+	"fmt"
+	"io"
+	"io/ioutil"
 	"log"
+	"math"
+	"net/http"
+	"net/url"
+	"os"
+	"strconv"
 	"time"
 
-	"github.com/cloudflare/cloudflare-go"
 	"github.com/micro/go-micro/config/options"
 	"github.com/micro/go-micro/store"
+	"github.com/pkg/errors"
 )
 
-var namespaceUUID string
+const (
+	apiBaseURL = "https://api.cloudflare.com/client/v4/"
+)
 
 type workersKV struct {
 	options.Options
-	api *cloudflare.API
+	// cf account id
+	account string
+	// cf api token
+	token string
+	// cf kv namespace
+	namespace string
+	// http client to use
+	httpClient *http.Client
 }
 
-// New returns a cloudflare Store implementation.
-// Options expects CF_API_TOKEN to a cloudflare API token scoped to Workers KV,
-// CF_ACCOUNT_ID to contain a string with your cloudflare account ID and
-// KV_NAMESPACE_ID to contain the namespace UUID for your KV storage.
-func New(opts ...options.Option) (store.Store, error) {
-	// Validate Options
-	options := options.NewOptions(opts...)
-	apiToken, ok := options.Values().Get("CF_API_TOKEN")
-	if !ok {
-		log.Fatal("Store: No CF_API_TOKEN passed as an option")
-	}
-	apiTokenString, ok := apiToken.(string)
-	if !ok {
-		log.Fatal("Store: Option CF_API_TOKEN contains a non-string")
-	}
-	accountID, ok := options.Values().Get("CF_ACCOUNT_ID")
-	if !ok {
-		log.Fatal("Store: No CF_ACCOUNT_ID passed as an option")
-	}
-	accountIDString, ok := accountID.(string)
-	if !ok {
-		log.Fatal("Store: Option CF_ACCOUNT_ID contains a non-string")
-	}
-	uuid, ok := options.Values().Get("KV_NAMESPACE_ID")
-	if !ok {
-		log.Fatal("Store: No KV_NAMESPACE_ID passed as an option")
-	}
-	namespaceUUID, ok = uuid.(string)
-	if !ok {
-		log.Fatal("Store: Option KV_NAMESPACE_ID contains a non-string")
-	}
-
-	// Create API client
-	api, err := cloudflare.NewWithAPIToken(apiTokenString, cloudflare.UsingAccount(accountIDString))
-	if err != nil {
-		return nil, err
-	}
-	return &workersKV{
-		Options: options,
-		api:     api,
-	}, nil
+// apiResponse is a cloudflare v4 api response
+type apiResponse struct {
+	Result []struct {
+		ID         string    `json:"id"`
+		Type       string    `json:"type"`
+		Name       string    `json:"name"`
+		Expiration string    `json:"expiration"`
+		Content    string    `json:"content"`
+		Proxiable  bool      `json:"proxiable"`
+		Proxied    bool      `json:"proxied"`
+		TTL        int       `json:"ttl"`
+		Priority   int       `json:"priority"`
+		Locked     bool      `json:"locked"`
+		ZoneID     string    `json:"zone_id"`
+		ZoneName   string    `json:"zone_name"`
+		ModifiedOn time.Time `json:"modified_on"`
+		CreatedOn  time.Time `json:"created_on"`
+	} `json:"result"`
+	Success bool         `json:"success"`
+	Errors  []apiMessage `json:"errors"`
+	// not sure Messages is ever populated?
+	Messages   []apiMessage `json:"messages"`
+	ResultInfo struct {
+		Page       int `json:"page"`
+		PerPage    int `json:"per_page"`
+		Count      int `json:"count"`
+		TotalCount int `json:"total_count"`
+	} `json:"result_info"`
 }
 
-// In the cloudflare workers KV implemention, Sync() doesn't guarantee
+// apiMessage is a Cloudflare v4 API Error
+type apiMessage struct {
+	Code    int    `json:"code"`
+	Message string `json:"message"`
+}
+
+// getOptions returns account id, token and namespace
+func getOptions() (string, string, string) {
+	accountID := os.Getenv("CF_ACCOUNT_ID")
+	apiToken := os.Getenv("CF_API_TOKEN")
+	namespace := os.Getenv("KV_NAMESPACE_ID")
+
+	return accountID, apiToken, namespace
+}
+
+func validateOptions(account, token, namespace string) {
+	if len(account) == 0 {
+		log.Fatal("Store: CF_ACCOUNT_ID is blank")
+	}
+
+	if len(token) == 0 {
+		log.Fatal("Store: CF_API_TOKEN is blank")
+	}
+
+	if len(namespace) == 0 {
+		log.Fatal("Store: KV_NAMESPACE_ID is blank")
+	}
+}
+
+// In the cloudflare workers KV implemention, List() doesn't guarantee
 // anything as the workers API is eventually consistent.
-func (w *workersKV) Sync() ([]*store.Record, error) {
-	response, err := w.api.ListWorkersKVs(context.Background(), namespaceUUID)
+func (w *workersKV) List() ([]*store.Record, error) {
+	ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
+	defer cancel()
+
+	path := fmt.Sprintf("accounts/%s/storage/kv/namespaces/%s/keys", w.account, w.namespace)
+
+	response, _, _, err := w.request(ctx, http.MethodGet, path, nil, make(http.Header))
 	if err != nil {
 		return nil, err
 	}
+
+	a := &apiResponse{}
+	if err := json.Unmarshal(response, a); err != nil {
+		return nil, err
+	}
+
+	if !a.Success {
+		messages := ""
+		for _, m := range a.Errors {
+			messages += strconv.Itoa(m.Code) + " " + m.Message + "\n"
+		}
+		return nil, errors.New(messages)
+	}
+
 	var keys []string
-	for _, r := range response.Result {
+
+	for _, r := range a.Result {
 		keys = append(keys, r.Name)
 	}
+
 	return w.Read(keys...)
 }
 
@@ -81,16 +137,30 @@ func (w *workersKV) Read(keys ...string) ([]*store.Record, error) {
 	defer cancel()
 
 	var records []*store.Record
+
 	for _, k := range keys {
-		v, err := w.api.ReadWorkersKV(ctx, namespaceUUID, k)
+		path := fmt.Sprintf("accounts/%s/storage/kv/namespaces/%s/values/%s", w.account, w.namespace, url.PathEscape(k))
+		response, headers, status, err := w.request(ctx, http.MethodGet, path, nil, make(http.Header))
 		if err != nil {
 			return records, err
 		}
-		records = append(records, &store.Record{
+		if status < 200 || status >= 300 {
+			return records, errors.New("Received unexpected Status " + strconv.Itoa(status) + string(response))
+		}
+		record := &store.Record{
 			Key:   k,
-			Value: v,
-		})
+			Value: response,
+		}
+		if expiry := headers.Get("Expiration"); len(expiry) != 0 {
+			expiryUnix, err := strconv.ParseInt(expiry, 10, 64)
+			if err != nil {
+				return records, err
+			}
+			record.Expiry = time.Until(time.Unix(expiryUnix, 0))
+		}
+		records = append(records, record)
 	}
+
 	return records, nil
 }
 
@@ -99,10 +169,34 @@ func (w *workersKV) Write(records ...*store.Record) error {
 	defer cancel()
 
 	for _, r := range records {
-		if _, err := w.api.WriteWorkersKV(ctx, namespaceUUID, r.Key, r.Value); err != nil {
+		path := fmt.Sprintf("accounts/%s/storage/kv/namespaces/%s/values/%s", w.account, w.namespace, url.PathEscape(r.Key))
+		if r.Expiry != 0 {
+			// Minimum cloudflare TTL is 60 Seconds
+			exp := int(math.Max(60, math.Round(r.Expiry.Seconds())))
+			path = path + "?expiration_ttl=" + strconv.Itoa(exp)
+		}
+
+		headers := make(http.Header)
+
+		resp, _, _, err := w.request(ctx, http.MethodPut, path, r.Value, headers)
+		if err != nil {
 			return err
 		}
+
+		a := &apiResponse{}
+		if err := json.Unmarshal(resp, a); err != nil {
+			return err
+		}
+
+		if !a.Success {
+			messages := ""
+			for _, m := range a.Errors {
+				messages += strconv.Itoa(m.Code) + " " + m.Message + "\n"
+			}
+			return errors.New(messages)
+		}
 	}
+
 	return nil
 }
 
@@ -111,9 +205,129 @@ func (w *workersKV) Delete(keys ...string) error {
 	defer cancel()
 
 	for _, k := range keys {
-		if _, err := w.api.DeleteWorkersKV(ctx, namespaceUUID, k); err != nil {
+		path := fmt.Sprintf("accounts/%s/storage/kv/namespaces/%s/values/%s", w.account, w.namespace, url.PathEscape(k))
+		resp, _, _, err := w.request(ctx, http.MethodDelete, path, nil, make(http.Header))
+		if err != nil {
 			return err
 		}
+
+		a := &apiResponse{}
+		if err := json.Unmarshal(resp, a); err != nil {
+			return err
+		}
+
+		if !a.Success {
+			messages := ""
+			for _, m := range a.Errors {
+				messages += strconv.Itoa(m.Code) + " " + m.Message + "\n"
+			}
+			return errors.New(messages)
+		}
 	}
+
 	return nil
 }
+
+func (w *workersKV) request(ctx context.Context, method, path string, body interface{}, headers http.Header) ([]byte, http.Header, int, error) {
+	var jsonBody []byte
+	var err error
+
+	if body != nil {
+		if paramBytes, ok := body.([]byte); ok {
+			jsonBody = paramBytes
+		} else {
+			jsonBody, err = json.Marshal(body)
+			if err != nil {
+				return nil, nil, 0, errors.Wrap(err, "error marshalling params to JSON")
+			}
+		}
+	} else {
+		jsonBody = nil
+	}
+
+	var reqBody io.Reader
+
+	if jsonBody != nil {
+		reqBody = bytes.NewReader(jsonBody)
+	}
+
+	req, err := http.NewRequestWithContext(ctx, method, apiBaseURL+path, reqBody)
+	for key, value := range headers {
+		req.Header[key] = value
+	}
+
+	// set token if it exists
+	if len(w.token) > 0 {
+		req.Header.Set("Authorization", "Bearer "+w.token)
+	}
+
+	// set the user agent to micro
+	req.Header.Set("User-Agent", "micro/1.0 (https://micro.mu)")
+
+	// Official cloudflare client does exponential backoff here
+	// TODO: retry and use util/backoff
+	resp, err := w.httpClient.Do(req)
+	if err != nil {
+		return nil, nil, 0, err
+	}
+	defer resp.Body.Close()
+
+	respBody, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return respBody, resp.Header, resp.StatusCode, err
+	}
+
+	return respBody, resp.Header, resp.StatusCode, nil
+}
+
+// New returns a cloudflare Store implementation.
+// Account ID, Token and Namespace must either be passed as options or
+// environment variables. If set as env vars we expect the following;
+// CF_API_TOKEN to a cloudflare API token scoped to Workers KV.
+// CF_ACCOUNT_ID to contain a string with your cloudflare account ID.
+// KV_NAMESPACE_ID to contain the namespace UUID for your KV storage.
+func NewStore(opts ...options.Option) store.Store {
+	// create new Options
+	options := options.NewOptions(opts...)
+
+	// get values from the environment
+	account, token, namespace := getOptions()
+
+	// set api token from options if exists
+	if apiToken, ok := options.Values().Get("CF_API_TOKEN"); ok {
+		tk, ok := apiToken.(string)
+		if !ok {
+			log.Fatal("Store: Option CF_API_TOKEN contains a non-string")
+		}
+		token = tk
+	}
+
+	// set account id from options if exists
+	if accountID, ok := options.Values().Get("CF_ACCOUNT_ID"); ok {
+		id, ok := accountID.(string)
+		if !ok {
+			log.Fatal("Store: Option CF_ACCOUNT_ID contains a non-string")
+		}
+		account = id
+	}
+
+	// set namespace from options if exists
+	if uuid, ok := options.Values().Get("KV_NAMESPACE_ID"); ok {
+		ns, ok := uuid.(string)
+		if !ok {
+			log.Fatal("Store: Option KV_NAMESPACE_ID contains a non-string")
+		}
+		namespace = ns
+	}
+
+	// validate options are not blank or log.Fatal
+	validateOptions(account, token, namespace)
+
+	return &workersKV{
+		account:    account,
+		namespace:  namespace,
+		token:      token,
+		Options:    options,
+		httpClient: &http.Client{},
+	}
+}

store/cloudflare/cloudflare_test.go

@@ -7,7 +7,6 @@ import (
 	"testing"
 	"time"
 
-	"github.com/micro/go-micro/config/options"
 	"github.com/micro/go-micro/store"
 )
 
@@ -21,25 +20,30 @@ func TestCloudflare(t *testing.T) {
 	randomK := strconv.Itoa(rand.Int())
 	randomV := strconv.Itoa(rand.Int())
 
-	wkv, err := New(
-		options.WithValue("CF_API_TOKEN", apiToken),
-		options.WithValue("CF_ACCOUNT_ID", accountID),
-		options.WithValue("KV_NAMESPACE_ID", kvID),
+	wkv := NewStore(
+		Token(apiToken),
+		Account(accountID),
+		Namespace(kvID),
 	)
 
+	records, err := wkv.List()
 	if err != nil {
-		t.Fatal(err.Error())
+		t.Fatalf("List: %s\n", err.Error())
+	} else {
+		t.Log("Listed " + strconv.Itoa(len(records)) + " records")
 	}
 
-	_, err = wkv.Sync()
-	if err != nil {
-		t.Fatalf("Sync: %s\n", err.Error())
-	}
-
-	err = wkv.Write(&store.Record{
-		Key:   randomK,
-		Value: []byte(randomV),
-	})
+	err = wkv.Write(
+		&store.Record{
+			Key:   randomK,
+			Value: []byte(randomV),
+		},
+		&store.Record{
+			Key:    "expirationtest",
+			Value:  []byte("This message will self destruct"),
+			Expiry: 75 * time.Second,
+		},
+	)
 	if err != nil {
 		t.Errorf("Write: %s", err.Error())
 	}
@@ -58,6 +62,23 @@ func TestCloudflare(t *testing.T) {
 		t.Errorf("Read: expected %s, got %s\n", randomK, string(r[0].Value))
 	}
 
+	r, err = wkv.Read("expirationtest")
+	if err != nil {
+		t.Errorf("Read: expirationtest should still exist")
+	}
+	if r[0].Expiry == 0 {
+		t.Error("Expected r to have an expiry")
+	} else {
+		t.Log(r[0].Expiry)
+	}
+
+	time.Sleep(20 * time.Second)
+	r, err = wkv.Read("expirationtest")
+	if err == nil && len(r) != 0 {
+		t.Error("Read: Managed to read expirationtest, but it should have expired")
+		t.Log(err, r[0].Key, string(r[0].Value), r[0].Expiry, len(r))
+	}
+
 	err = wkv.Delete(randomK)
 	if err != nil {
 		t.Errorf("Delete: %s\n", err.Error())

store/cloudflare/options.go

@@ -0,0 +1,23 @@
+package cloudflare
+
+import (
+	"github.com/micro/go-micro/config/options"
+)
+
+// Token sets the cloudflare api token
+func Token(t string) options.Option {
+	// TODO: change to store.cf.api_token
+	return options.WithValue("CF_API_TOKEN", t)
+}
+
+// Account sets the cloudflare account id
+func Account(id string) options.Option {
+	// TODO: change to store.cf.account_id
+	return options.WithValue("CF_ACCOUNT_ID", id)
+}
+
+// Namespace sets the KV namespace
+func Namespace(ns string) options.Option {
+	// TODO: change to store.cf.namespace
+	return options.WithValue("KV_NAMESPACE_ID", ns)
+}

store/etcd/etcd.go

@@ -68,7 +68,7 @@ func (e *ekv) Write(records ...*store.Record) error {
 	return gerr
 }
 
-func (e *ekv) Sync() ([]*store.Record, error) {
+func (e *ekv) List() ([]*store.Record, error) {
 	keyval, err := e.kv.Get(context.Background(), "/", client.WithPrefix())
 	if err != nil {
 		return nil, err

store/memory/memory.go

@@ -21,7 +21,7 @@ type memoryRecord struct {
 	c time.Time
 }
 
-func (m *memoryStore) Sync() ([]*store.Record, error) {
+func (m *memoryStore) List() ([]*store.Record, error) {
 	m.RLock()
 	defer m.RUnlock()
 

store/options.go

@@ -4,7 +4,7 @@ import (
 	"github.com/micro/go-micro/config/options"
 )
 
-// Set the nodes used to back the store
+// Nodes is a list of nodes used to back the store
 func Nodes(a ...string) options.Option {
 	return options.WithValue("store.nodes", a)
 }
@@ -13,3 +13,9 @@ func Nodes(a ...string) options.Option {
 func Prefix(p string) options.Option {
 	return options.WithValue("store.prefix", p)
 }
+
+// Namespace offers a way to have multiple isolated
+// stores in the same backend, if supported.
+func Namespace(n string) options.Option {
+	return options.WithValue("store.namespace", n)
+}

store/postgresql/postgresql.go

@@ -0,0 +1,248 @@
+// Package postgresql implements a micro Store backed by sql
+package postgresql
+
+import (
+	"database/sql"
+	"fmt"
+	"strings"
+	"time"
+	"unicode"
+
+	"github.com/lib/pq"
+	"github.com/pkg/errors"
+
+	"github.com/micro/go-micro/config/options"
+	"github.com/micro/go-micro/store"
+)
+
+// DefaultNamespace is the namespace that the sql store
+// will use if no namespace is provided.
+const DefaultNamespace = "micro"
+
+type sqlStore struct {
+	db *sql.DB
+
+	table string
+	options.Options
+}
+
+// List all the known records
+func (s *sqlStore) List() ([]*store.Record, error) {
+	q, err := s.db.Prepare(fmt.Sprintf("SELECT key, value, expiry FROM micro.%s;", s.table))
+	if err != nil {
+		return nil, err
+	}
+	var records []*store.Record
+	var timehelper pq.NullTime
+	rows, err := q.Query()
+	if err != nil {
+		if err == sql.ErrNoRows {
+			return records, nil
+		}
+		return nil, err
+	}
+	defer rows.Close()
+	for rows.Next() {
+		record := &store.Record{}
+		if err := rows.Scan(&record.Key, &record.Value, &timehelper); err != nil {
+			return records, err
+		}
+		if timehelper.Valid {
+			if timehelper.Time.Before(time.Now()) {
+				// record has expired
+				go s.Delete(record.Key)
+			} else {
+				record.Expiry = time.Until(timehelper.Time)
+				records = append(records, record)
+			}
+		} else {
+			records = append(records, record)
+		}
+
+	}
+	rowErr := rows.Close()
+	if rowErr != nil {
+		// transaction rollback or something
+		return records, rowErr
+	}
+	if err := rows.Err(); err != nil {
+		return records, err
+	}
+	return records, nil
+}
+
+// Read all records with keys
+func (s *sqlStore) Read(keys ...string) ([]*store.Record, error) {
+	q, err := s.db.Prepare(fmt.Sprintf("SELECT key, value, expiry FROM micro.%s WHERE key = $1;", s.table))
+	if err != nil {
+		return nil, err
+	}
+	var records []*store.Record
+	var timehelper pq.NullTime
+	for _, key := range keys {
+		row := q.QueryRow(key)
+		record := &store.Record{}
+		if err := row.Scan(&record.Key, &record.Value, &timehelper); err != nil {
+			if err == sql.ErrNoRows {
+				return records, store.ErrNotFound
+			}
+			return records, err
+		}
+		if timehelper.Valid {
+			if timehelper.Time.Before(time.Now()) {
+				// record has expired
+				go s.Delete(key)
+				return records, store.ErrNotFound
+			}
+			record.Expiry = time.Until(timehelper.Time)
+			records = append(records, record)
+		} else {
+			records = append(records, record)
+		}
+	}
+	return records, nil
+}
+
+// Write records
+func (s *sqlStore) Write(rec ...*store.Record) error {
+	q, err := s.db.Prepare(fmt.Sprintf(`INSERT INTO micro.%s(key, value, expiry)
+		VALUES ($1, $2::bytea, $3)
+		ON CONFLICT (key)
+		DO UPDATE
+		SET value = EXCLUDED.value, expiry = EXCLUDED.expiry;`, s.table))
+	if err != nil {
+		return err
+	}
+	for _, r := range rec {
+		var err error
+		if r.Expiry != 0 {
+			_, err = q.Exec(r.Key, r.Value, time.Now().Add(r.Expiry))
+		} else {
+			_, err = q.Exec(r.Key, r.Value, nil)
+		}
+		if err != nil {
+			return errors.Wrap(err, "Couldn't insert record "+r.Key)
+		}
+	}
+
+	return nil
+}
+
+// Delete records with keys
+func (s *sqlStore) Delete(keys ...string) error {
+	q, err := s.db.Prepare(fmt.Sprintf("DELETE FROM micro.%s WHERE key = $1;", s.table))
+	if err != nil {
+		return err
+	}
+	for _, key := range keys {
+		result, err := q.Exec(key)
+		if err != nil {
+			return err
+		}
+		_, err = result.RowsAffected()
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (s *sqlStore) initDB(options options.Options) error {
+	// Get the store.namespace option, or use sql.DefaultNamespace
+	namespaceOpt, found := options.Values().Get("store.namespace")
+	if !found {
+		s.table = DefaultNamespace
+	} else {
+		if namespace, ok := namespaceOpt.(string); ok {
+			s.table = namespace
+		} else {
+			return errors.New("store.namespace option must be a string")
+		}
+	}
+
+	// Create "micro" schema
+	schema, err := s.db.Prepare("CREATE SCHEMA IF NOT EXISTS micro ;")
+	if err != nil {
+		return err
+	}
+	_, err = schema.Exec()
+	if err != nil {
+		return errors.Wrap(err, "Couldn't create Schema")
+	}
+
+	// Create a table for the Store namespace
+	tableq, err := s.db.Prepare(fmt.Sprintf(`CREATE TABLE IF NOT EXISTS micro.%s
+	(
+		key text COLLATE "default" NOT NULL,
+		value bytea,
+		expiry timestamp with time zone,
+		CONSTRAINT %s_pkey PRIMARY KEY (key)
+	);`, s.table, s.table))
+	_, err = tableq.Exec()
+	if err != nil {
+		return errors.Wrap(err, "Couldn't create table")
+	}
+
+	return nil
+}
+
+// New returns a new micro Store backed by sql
+func New(opts ...options.Option) (store.Store, error) {
+	options := options.NewOptions(opts...)
+	driver, dataSourceName, err := validateOptions(options)
+	if err != nil {
+		return nil, err
+	}
+	if !strings.Contains(dataSourceName, " ") {
+		dataSourceName = fmt.Sprintf("host=%s", dataSourceName)
+	}
+	db, err := sql.Open(driver, dataSourceName)
+	if err != nil {
+		return nil, err
+	}
+	if err := db.Ping(); err != nil {
+		return nil, err
+	}
+	s := &sqlStore{
+		db: db,
+	}
+
+	return s, s.initDB(options)
+}
+
+// validateOptions checks whether the provided options are valid, then returns the driver
+// and data source name.
+func validateOptions(options options.Options) (driver, dataSourceName string, err error) {
+	driverOpt, found := options.Values().Get("store.sql.driver")
+	if !found {
+		return "", "", errors.New("No store.sql.driver option specified")
+	}
+	nodesOpt, found := options.Values().Get("store.nodes")
+	if !found {
+		return "", "", errors.New("No store.nodes option specified (expected a database connection string)")
+	}
+	driver, ok := driverOpt.(string)
+	if !ok {
+		return "", "", errors.New("store.sql.driver option must be a string")
+	}
+	nodes, ok := nodesOpt.([]string)
+	if !ok {
+		return "", "", errors.New("store.nodes option must be a []string")
+	}
+	if len(nodes) != 1 {
+		return "", "", errors.New("expected only 1 store.nodes option")
+	}
+	namespaceOpt, found := options.Values().Get("store.namespace")
+	if found {
+		namespace, ok := namespaceOpt.(string)
+		if !ok {
+			return "", "", errors.New("store.namespace must me a string")
+		}
+		for _, r := range namespace {
+			if !unicode.IsLetter(r) {
+				return "", "", errors.New("store.namespace must only contain letters")
+			}
+		}
+	}
+	return driver, nodes[0], nil
+}

store/postgresql/postgresql_test.go

@@ -0,0 +1,95 @@
+package postgresql
+
+import (
+	"database/sql"
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/kr/pretty"
+	"github.com/micro/go-micro/store"
+)
+
+func TestSQL(t *testing.T) {
+	connection := fmt.Sprintf(
+		"host=%s port=%d user=%s sslmode=disable dbname=%s",
+		"localhost",
+		5432,
+		"jake",
+		"test",
+	)
+	db, err := sql.Open("postgres", connection)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if err := db.Ping(); err != nil {
+		t.Skip(err)
+	}
+	db.Close()
+
+	sqlStore, err := New(
+		store.Namespace("testsql"),
+		store.Nodes(connection),
+	)
+	if err != nil {
+		t.Fatal(err.Error())
+	}
+
+	records, err := sqlStore.List()
+	if err != nil {
+		t.Error(err)
+	} else {
+		t.Logf("%# v\n", pretty.Formatter(records))
+	}
+
+	err = sqlStore.Write(
+		&store.Record{
+			Key:   "test",
+			Value: []byte("foo"),
+		},
+		&store.Record{
+			Key:   "bar",
+			Value: []byte("baz"),
+		},
+		&store.Record{
+			Key:   "qux",
+			Value: []byte("aasad"),
+		},
+	)
+	if err != nil {
+		t.Error(err)
+	}
+	err = sqlStore.Delete("qux")
+	if err != nil {
+		t.Error(err)
+	}
+
+	err = sqlStore.Write(&store.Record{
+		Key:    "test",
+		Value:  []byte("bar"),
+		Expiry: time.Minute,
+	})
+	if err != nil {
+		t.Error(err)
+	}
+
+	records, err = sqlStore.Read("test")
+	if err != nil {
+		t.Error(err)
+	} else {
+		t.Logf("%# v\n", pretty.Formatter(records))
+		if string(records[0].Value) != "bar" {
+			t.Error("Expected bar, got ", string(records[0].Value))
+		}
+	}
+
+	time.Sleep(61 * time.Second)
+	records, err = sqlStore.Read("test")
+	if err == nil {
+		t.Error("Key test should have expired")
+	} else {
+		if err != store.ErrNotFound {
+			t.Error(err)
+		}
+	}
+}

store/service/handler/handler.go

@@ -55,19 +55,19 @@ func (s *Store) Delete(ctx context.Context, req *pb.DeleteRequest, rsp *pb.Delet
 	return nil
 }
 
-func (s *Store) Sync(ctx context.Context, req *pb.SyncRequest, stream pb.Store_SyncStream) error {
+func (s *Store) List(ctx context.Context, req *pb.ListRequest, stream pb.Store_ListStream) error {
 	var vals []*store.Record
 	var err error
 
 	if len(req.Key) > 0 {
 		vals, err = s.Store.Read(req.Key)
 	} else {
-		vals, err = s.Store.Sync()
+		vals, err = s.Store.List()
 	}
 	if err != nil {
 		return errors.InternalServerError("go.micro.store", err.Error())
 	}
-	rsp := new(pb.SyncResponse)
+	rsp := new(pb.ListResponse)
 
 	// TODO: batch sync
 	for _, val := range vals {

store/service/proto/store.micro.go

@@ -34,10 +34,10 @@ var _ server.Option
 // Client API for Store service
 
 type StoreService interface {
+	List(ctx context.Context, in *ListRequest, opts ...client.CallOption) (Store_ListService, error)
 	Read(ctx context.Context, in *ReadRequest, opts ...client.CallOption) (*ReadResponse, error)
 	Write(ctx context.Context, in *WriteRequest, opts ...client.CallOption) (*WriteResponse, error)
 	Delete(ctx context.Context, in *DeleteRequest, opts ...client.CallOption) (*DeleteResponse, error)
-	Sync(ctx context.Context, in *SyncRequest, opts ...client.CallOption) (Store_SyncService, error)
 }
 
 type storeService struct {
@@ -58,6 +58,50 @@ func NewStoreService(name string, c client.Client) StoreService {
 	}
 }
 
+func (c *storeService) List(ctx context.Context, in *ListRequest, opts ...client.CallOption) (Store_ListService, error) {
+	req := c.c.NewRequest(c.name, "Store.List", &ListRequest{})
+	stream, err := c.c.Stream(ctx, req, opts...)
+	if err != nil {
+		return nil, err
+	}
+	if err := stream.Send(in); err != nil {
+		return nil, err
+	}
+	return &storeServiceList{stream}, nil
+}
+
+type Store_ListService interface {
+	SendMsg(interface{}) error
+	RecvMsg(interface{}) error
+	Close() error
+	Recv() (*ListResponse, error)
+}
+
+type storeServiceList struct {
+	stream client.Stream
+}
+
+func (x *storeServiceList) Close() error {
+	return x.stream.Close()
+}
+
+func (x *storeServiceList) SendMsg(m interface{}) error {
+	return x.stream.Send(m)
+}
+
+func (x *storeServiceList) RecvMsg(m interface{}) error {
+	return x.stream.Recv(m)
+}
+
+func (x *storeServiceList) Recv() (*ListResponse, error) {
+	m := new(ListResponse)
+	err := x.stream.Recv(m)
+	if err != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
 func (c *storeService) Read(ctx context.Context, in *ReadRequest, opts ...client.CallOption) (*ReadResponse, error) {
 	req := c.c.NewRequest(c.name, "Store.Read", in)
 	out := new(ReadResponse)
@@ -88,65 +132,21 @@ func (c *storeService) Delete(ctx context.Context, in *DeleteRequest, opts ...cl
 	return out, nil
 }
 
-func (c *storeService) Sync(ctx context.Context, in *SyncRequest, opts ...client.CallOption) (Store_SyncService, error) {
-	req := c.c.NewRequest(c.name, "Store.Sync", &SyncRequest{})
-	stream, err := c.c.Stream(ctx, req, opts...)
-	if err != nil {
-		return nil, err
-	}
-	if err := stream.Send(in); err != nil {
-		return nil, err
-	}
-	return &storeServiceSync{stream}, nil
-}
-
-type Store_SyncService interface {
-	SendMsg(interface{}) error
-	RecvMsg(interface{}) error
-	Close() error
-	Recv() (*SyncResponse, error)
-}
-
-type storeServiceSync struct {
-	stream client.Stream
-}
-
-func (x *storeServiceSync) Close() error {
-	return x.stream.Close()
-}
-
-func (x *storeServiceSync) SendMsg(m interface{}) error {
-	return x.stream.Send(m)
-}
-
-func (x *storeServiceSync) RecvMsg(m interface{}) error {
-	return x.stream.Recv(m)
-}
-
-func (x *storeServiceSync) Recv() (*SyncResponse, error) {
-	m := new(SyncResponse)
-	err := x.stream.Recv(m)
-	if err != nil {
-		return nil, err
-	}
-	return m, nil
-}
-
 // Server API for Store service
 
 type StoreHandler interface {
+	List(context.Context, *ListRequest, Store_ListStream) error
 	Read(context.Context, *ReadRequest, *ReadResponse) error
 	Write(context.Context, *WriteRequest, *WriteResponse) error
 	Delete(context.Context, *DeleteRequest, *DeleteResponse) error
-	Sync(context.Context, *SyncRequest, Store_SyncStream) error
 }
 
 func RegisterStoreHandler(s server.Server, hdlr StoreHandler, opts ...server.HandlerOption) error {
 	type store interface {
+		List(ctx context.Context, stream server.Stream) error
 		Read(ctx context.Context, in *ReadRequest, out *ReadResponse) error
 		Write(ctx context.Context, in *WriteRequest, out *WriteResponse) error
 		Delete(ctx context.Context, in *DeleteRequest, out *DeleteResponse) error
-		Sync(ctx context.Context, stream server.Stream) error
 	}
 	type Store struct {
 		store
@@ -159,6 +159,41 @@ type storeHandler struct {
 	StoreHandler
 }
 
+func (h *storeHandler) List(ctx context.Context, stream server.Stream) error {
+	m := new(ListRequest)
+	if err := stream.Recv(m); err != nil {
+		return err
+	}
+	return h.StoreHandler.List(ctx, m, &storeListStream{stream})
+}
+
+type Store_ListStream interface {
+	SendMsg(interface{}) error
+	RecvMsg(interface{}) error
+	Close() error
+	Send(*ListResponse) error
+}
+
+type storeListStream struct {
+	stream server.Stream
+}
+
+func (x *storeListStream) Close() error {
+	return x.stream.Close()
+}
+
+func (x *storeListStream) SendMsg(m interface{}) error {
+	return x.stream.Send(m)
+}
+
+func (x *storeListStream) RecvMsg(m interface{}) error {
+	return x.stream.Recv(m)
+}
+
+func (x *storeListStream) Send(m *ListResponse) error {
+	return x.stream.Send(m)
+}
+
 func (h *storeHandler) Read(ctx context.Context, in *ReadRequest, out *ReadResponse) error {
 	return h.StoreHandler.Read(ctx, in, out)
 }
@@ -170,38 +205,3 @@ func (h *storeHandler) Write(ctx context.Context, in *WriteRequest, out *WriteRe
 func (h *storeHandler) Delete(ctx context.Context, in *DeleteRequest, out *DeleteResponse) error {
 	return h.StoreHandler.Delete(ctx, in, out)
 }
-
-func (h *storeHandler) Sync(ctx context.Context, stream server.Stream) error {
-	m := new(SyncRequest)
-	if err := stream.Recv(m); err != nil {
-		return err
-	}
-	return h.StoreHandler.Sync(ctx, m, &storeSyncStream{stream})
-}
-
-type Store_SyncStream interface {
-	SendMsg(interface{}) error
-	RecvMsg(interface{}) error
-	Close() error
-	Send(*SyncResponse) error
-}
-
-type storeSyncStream struct {
-	stream server.Stream
-}
-
-func (x *storeSyncStream) Close() error {
-	return x.stream.Close()
-}
-
-func (x *storeSyncStream) SendMsg(m interface{}) error {
-	return x.stream.Send(m)
-}
-
-func (x *storeSyncStream) RecvMsg(m interface{}) error {
-	return x.stream.Recv(m)
-}
-
-func (x *storeSyncStream) Send(m *SyncResponse) error {
-	return x.stream.Send(m)
-}

store/service/proto/store.pb.go

@@ -298,7 +298,7 @@ func (m *DeleteResponse) XXX_DiscardUnknown() {
 
 var xxx_messageInfo_DeleteResponse proto.InternalMessageInfo
 
-type SyncRequest struct {
+type ListRequest struct {
 	// optional key
 	Key                  string   `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"`
 	XXX_NoUnkeyedLiteral struct{} `json:"-"`
@@ -306,71 +306,71 @@ type SyncRequest struct {
 	XXX_sizecache        int32    `json:"-"`
 }
 
-func (m *SyncRequest) Reset()         { *m = SyncRequest{} }
-func (m *SyncRequest) String() string { return proto.CompactTextString(m) }
-func (*SyncRequest) ProtoMessage()    {}
-func (*SyncRequest) Descriptor() ([]byte, []int) {
+func (m *ListRequest) Reset()         { *m = ListRequest{} }
+func (m *ListRequest) String() string { return proto.CompactTextString(m) }
+func (*ListRequest) ProtoMessage()    {}
+func (*ListRequest) Descriptor() ([]byte, []int) {
 	return fileDescriptor_f84ccc98e143ed3e, []int{7}
 }
 
-func (m *SyncRequest) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_SyncRequest.Unmarshal(m, b)
+func (m *ListRequest) XXX_Unmarshal(b []byte) error {
+	return xxx_messageInfo_ListRequest.Unmarshal(m, b)
 }
-func (m *SyncRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_SyncRequest.Marshal(b, m, deterministic)
+func (m *ListRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	return xxx_messageInfo_ListRequest.Marshal(b, m, deterministic)
 }
-func (m *SyncRequest) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_SyncRequest.Merge(m, src)
+func (m *ListRequest) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_ListRequest.Merge(m, src)
 }
-func (m *SyncRequest) XXX_Size() int {
-	return xxx_messageInfo_SyncRequest.Size(m)
+func (m *ListRequest) XXX_Size() int {
+	return xxx_messageInfo_ListRequest.Size(m)
 }
-func (m *SyncRequest) XXX_DiscardUnknown() {
-	xxx_messageInfo_SyncRequest.DiscardUnknown(m)
+func (m *ListRequest) XXX_DiscardUnknown() {
+	xxx_messageInfo_ListRequest.DiscardUnknown(m)
 }
 
-var xxx_messageInfo_SyncRequest proto.InternalMessageInfo
+var xxx_messageInfo_ListRequest proto.InternalMessageInfo
 
-func (m *SyncRequest) GetKey() string {
+func (m *ListRequest) GetKey() string {
 	if m != nil {
 		return m.Key
 	}
 	return ""
 }
 
-type SyncResponse struct {
+type ListResponse struct {
 	Records              []*Record `protobuf:"bytes,1,rep,name=records,proto3" json:"records,omitempty"`
 	XXX_NoUnkeyedLiteral struct{}  `json:"-"`
 	XXX_unrecognized     []byte    `json:"-"`
 	XXX_sizecache        int32     `json:"-"`
 }
 
-func (m *SyncResponse) Reset()         { *m = SyncResponse{} }
-func (m *SyncResponse) String() string { return proto.CompactTextString(m) }
-func (*SyncResponse) ProtoMessage()    {}
-func (*SyncResponse) Descriptor() ([]byte, []int) {
+func (m *ListResponse) Reset()         { *m = ListResponse{} }
+func (m *ListResponse) String() string { return proto.CompactTextString(m) }
+func (*ListResponse) ProtoMessage()    {}
+func (*ListResponse) Descriptor() ([]byte, []int) {
 	return fileDescriptor_f84ccc98e143ed3e, []int{8}
 }
 
-func (m *SyncResponse) XXX_Unmarshal(b []byte) error {
-	return xxx_messageInfo_SyncResponse.Unmarshal(m, b)
+func (m *ListResponse) XXX_Unmarshal(b []byte) error {
+	return xxx_messageInfo_ListResponse.Unmarshal(m, b)
 }
-func (m *SyncResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
-	return xxx_messageInfo_SyncResponse.Marshal(b, m, deterministic)
+func (m *ListResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	return xxx_messageInfo_ListResponse.Marshal(b, m, deterministic)
 }
-func (m *SyncResponse) XXX_Merge(src proto.Message) {
-	xxx_messageInfo_SyncResponse.Merge(m, src)
+func (m *ListResponse) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_ListResponse.Merge(m, src)
 }
-func (m *SyncResponse) XXX_Size() int {
-	return xxx_messageInfo_SyncResponse.Size(m)
+func (m *ListResponse) XXX_Size() int {
+	return xxx_messageInfo_ListResponse.Size(m)
 }
-func (m *SyncResponse) XXX_DiscardUnknown() {
-	xxx_messageInfo_SyncResponse.DiscardUnknown(m)
+func (m *ListResponse) XXX_DiscardUnknown() {
+	xxx_messageInfo_ListResponse.DiscardUnknown(m)
 }
 
-var xxx_messageInfo_SyncResponse proto.InternalMessageInfo
+var xxx_messageInfo_ListResponse proto.InternalMessageInfo
 
-func (m *SyncResponse) GetRecords() []*Record {
+func (m *ListResponse) GetRecords() []*Record {
 	if m != nil {
 		return m.Records
 	}
@@ -385,8 +385,8 @@ func init() {
 	proto.RegisterType((*WriteResponse)(nil), "go.micro.store.WriteResponse")
 	proto.RegisterType((*DeleteRequest)(nil), "go.micro.store.DeleteRequest")
 	proto.RegisterType((*DeleteResponse)(nil), "go.micro.store.DeleteResponse")
-	proto.RegisterType((*SyncRequest)(nil), "go.micro.store.SyncRequest")
-	proto.RegisterType((*SyncResponse)(nil), "go.micro.store.SyncResponse")
+	proto.RegisterType((*ListRequest)(nil), "go.micro.store.ListRequest")
+	proto.RegisterType((*ListResponse)(nil), "go.micro.store.ListResponse")
 }
 
 func init() {
@@ -395,27 +395,27 @@ func init() {
 
 var fileDescriptor_f84ccc98e143ed3e = []byte{
 	// 333 bytes of a gzipped FileDescriptorProto
-	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x92, 0xcb, 0x6e, 0xf2, 0x30,
-	0x10, 0x85, 0x09, 0x81, 0xfc, 0x62, 0xb8, 0xfc, 0x68, 0x54, 0xa1, 0x88, 0xde, 0xd2, 0x74, 0x93,
-	0x4d, 0x03, 0xa2, 0x2f, 0x50, 0xa9, 0x17, 0xb5, 0x5b, 0xb3, 0xe8, 0x9a, 0x86, 0x11, 0x8a, 0xa0,
-	0x98, 0x3a, 0x01, 0x35, 0x2f, 0xd4, 0xe7, 0xac, 0x6c, 0x27, 0x69, 0x90, 0x41, 0xaa, 0xba, 0x1b,
-	0x7b, 0xce, 0x1c, 0x9f, 0xf9, 0x64, 0x08, 0xdf, 0xe3, 0x48, 0xf0, 0xd1, 0x82, 0xdf, 0xe8, 0x22,
-	0x49, 0xb9, 0xa0, 0x51, 0x42, 0x62, 0x17, 0x47, 0x34, 0xda, 0x08, 0x9e, 0xe6, 0x77, 0xa1, 0xaa,
-	0xb1, 0xb7, 0xe0, 0x7a, 0x24, 0x54, 0xb7, 0xfe, 0x33, 0x38, 0x8c, 0x22, 0x2e, 0xe6, 0xd8, 0x07,
-	0x7b, 0x49, 0x99, 0x6b, 0x79, 0x56, 0xd0, 0x62, 0xb2, 0xc4, 0x13, 0x68, 0xee, 0x66, 0xab, 0x2d,
-	0xb9, 0x75, 0xcf, 0x0a, 0x3a, 0x4c, 0x1f, 0x70, 0x00, 0x0e, 0x7d, 0x6e, 0x62, 0x91, 0xb9, 0xb6,
-	0x67, 0x05, 0x36, 0xcb, 0x4f, 0xfe, 0x15, 0xb4, 0x19, 0xcd, 0xe6, 0x8c, 0x3e, 0xb6, 0x94, 0xa4,
-	0x88, 0xd0, 0x58, 0x52, 0x96, 0xb8, 0x96, 0x67, 0x07, 0x2d, 0xa6, 0x6a, 0xff, 0x0e, 0x3a, 0x5a,
-	0x92, 0x6c, 0xf8, 0x3a, 0x21, 0x1c, 0xc3, 0x3f, 0xa1, 0x1e, 0xd7, 0xb2, 0xf6, 0x64, 0x10, 0xee,
-	0xc7, 0x0b, 0x75, 0x36, 0x56, 0xc8, 0xa4, 0xc3, 0xab, 0x88, 0x53, 0x2a, 0x5e, 0xa9, 0x38, 0xd4,
-	0x7f, 0xe7, 0xf0, 0x1f, 0xba, 0xb9, 0x83, 0x0e, 0xe1, 0x5f, 0x43, 0xf7, 0x81, 0x56, 0xf4, 0xe3,
-	0x79, 0x28, 0x79, 0x1f, 0x7a, 0x85, 0x28, 0x1f, 0xbb, 0x84, 0xf6, 0x34, 0x5b, 0x47, 0xc5, 0x90,
-	0x41, 0x4f, 0x46, 0xd5, 0x82, 0xbf, 0x2e, 0x3b, 0xf9, 0xaa, 0x43, 0x73, 0x2a, 0x3b, 0x78, 0x0f,
-	0x0d, 0x09, 0x0e, 0x4f, 0xcd, 0x91, 0x92, 0xf8, 0xf0, 0xec, 0x70, 0x33, 0xcf, 0x5b, 0xc3, 0x27,
-	0x68, 0xaa, 0xcd, 0xd1, 0x10, 0x56, 0x91, 0x0e, 0xcf, 0x8f, 0x74, 0x4b, 0x9f, 0x17, 0x70, 0x34,
-	0x0b, 0x34, 0xa4, 0x7b, 0x20, 0x87, 0x17, 0xc7, 0xda, 0xa5, 0xd5, 0x23, 0x34, 0x24, 0x23, 0x73,
-	0xaf, 0x0a, 0x5a, 0x73, 0xaf, 0x2a, 0x56, 0xbf, 0x36, 0xb6, 0xde, 0x1c, 0xf5, 0xb7, 0x6f, 0xbf,
-	0x03, 0x00, 0x00, 0xff, 0xff, 0x30, 0xc8, 0x99, 0x52, 0x0d, 0x03, 0x00, 0x00,
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x92, 0x4d, 0x4f, 0xc2, 0x40,
+	0x10, 0x86, 0x59, 0x0a, 0x35, 0x0c, 0x1f, 0x92, 0x89, 0x21, 0x0d, 0x7e, 0xd5, 0x7a, 0xe9, 0xc5,
+	0x42, 0xf0, 0x0f, 0x98, 0xf8, 0x11, 0x4d, 0x3c, 0xad, 0x07, 0xcf, 0x08, 0x13, 0xd2, 0x80, 0x2e,
+	0xee, 0x16, 0x62, 0xff, 0x90, 0xbf, 0xd3, 0xec, 0x6e, 0xab, 0xc5, 0x42, 0x62, 0xbc, 0xcd, 0xee,
+	0xbc, 0xf3, 0xec, 0xdb, 0x79, 0x0b, 0xd1, 0x6b, 0x3c, 0x91, 0x62, 0x30, 0x13, 0x17, 0xb6, 0x50,
+	0x89, 0x90, 0x34, 0x50, 0x24, 0xd7, 0xf1, 0x84, 0x06, 0x4b, 0x29, 0x92, 0xec, 0x2e, 0x32, 0x35,
+	0x76, 0x66, 0xc2, 0x8e, 0x44, 0xe6, 0x36, 0xb8, 0x07, 0x97, 0xd3, 0x44, 0xc8, 0x29, 0x76, 0xc1,
+	0x99, 0x53, 0xea, 0x31, 0x9f, 0x85, 0x0d, 0xae, 0x4b, 0x3c, 0x80, 0xfa, 0x7a, 0xbc, 0x58, 0x91,
+	0x57, 0xf5, 0x59, 0xd8, 0xe2, 0xf6, 0x80, 0x3d, 0x70, 0xe9, 0x63, 0x19, 0xcb, 0xd4, 0x73, 0x7c,
+	0x16, 0x3a, 0x3c, 0x3b, 0x05, 0x67, 0xd0, 0xe4, 0x34, 0x9e, 0x72, 0x7a, 0x5f, 0x91, 0x4a, 0x10,
+	0xa1, 0x36, 0xa7, 0x54, 0x79, 0xcc, 0x77, 0xc2, 0x06, 0x37, 0x75, 0x70, 0x05, 0x2d, 0x2b, 0x51,
+	0x4b, 0xf1, 0xa6, 0x08, 0x87, 0xb0, 0x27, 0xcd, 0xe3, 0x56, 0xd6, 0x1c, 0xf5, 0xa2, 0x4d, 0x7b,
+	0x91, 0xf5, 0xc6, 0x73, 0x99, 0x26, 0x3c, 0xcb, 0x38, 0xa1, 0xfc, 0x95, 0x02, 0xa1, 0xfa, 0x37,
+	0xc2, 0x3e, 0xb4, 0x33, 0x82, 0x35, 0x11, 0x9c, 0x43, 0xfb, 0x86, 0x16, 0xf4, 0xc3, 0xdc, 0xe6,
+	0xbc, 0x0b, 0x9d, 0x5c, 0x94, 0x8d, 0x9d, 0x42, 0xf3, 0x31, 0x56, 0x49, 0x3e, 0x54, 0xda, 0x9e,
+	0xb6, 0x6a, 0x05, 0xff, 0xfd, 0xd8, 0xd1, 0x67, 0x15, 0xea, 0x4f, 0xba, 0x83, 0xb7, 0x50, 0xd3,
+	0x2c, 0x3c, 0xfc, 0x3d, 0x52, 0xb0, 0xd0, 0x3f, 0xda, 0xde, 0xcc, 0xfc, 0x56, 0x86, 0x0c, 0xaf,
+	0xa1, 0xa6, 0xf7, 0x5f, 0xc6, 0x14, 0x82, 0x2b, 0x63, 0x8a, 0x91, 0x05, 0x15, 0xbc, 0x83, 0xba,
+	0x59, 0x20, 0x96, 0x84, 0xc5, 0x64, 0xfa, 0xc7, 0x3b, 0xba, 0xdf, 0x9c, 0x07, 0x70, 0xed, 0x4a,
+	0xb1, 0x24, 0xdd, 0xc8, 0xa3, 0x7f, 0xb2, 0xab, 0x9d, 0xa3, 0x5e, 0x5c, 0xf3, 0x6f, 0x5f, 0x7e,
+	0x05, 0x00, 0x00, 0xff, 0xff, 0x30, 0x48, 0x25, 0x2d, 0x0d, 0x03, 0x00, 0x00,
 }
 
 // Reference imports to suppress errors if they are not otherwise used.
@@ -430,10 +430,10 @@ const _ = grpc.SupportPackageIsVersion4
 //
 // For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream.
 type StoreClient interface {
+	List(ctx context.Context, in *ListRequest, opts ...grpc.CallOption) (Store_ListClient, error)
 	Read(ctx context.Context, in *ReadRequest, opts ...grpc.CallOption) (*ReadResponse, error)
 	Write(ctx context.Context, in *WriteRequest, opts ...grpc.CallOption) (*WriteResponse, error)
 	Delete(ctx context.Context, in *DeleteRequest, opts ...grpc.CallOption) (*DeleteResponse, error)
-	Sync(ctx context.Context, in *SyncRequest, opts ...grpc.CallOption) (Store_SyncClient, error)
 }
 
 type storeClient struct {
@@ -444,6 +444,38 @@ func NewStoreClient(cc *grpc.ClientConn) StoreClient {
 	return &storeClient{cc}
 }
 
+func (c *storeClient) List(ctx context.Context, in *ListRequest, opts ...grpc.CallOption) (Store_ListClient, error) {
+	stream, err := c.cc.NewStream(ctx, &_Store_serviceDesc.Streams[0], "/go.micro.store.Store/List", opts...)
+	if err != nil {
+		return nil, err
+	}
+	x := &storeListClient{stream}
+	if err := x.ClientStream.SendMsg(in); err != nil {
+		return nil, err
+	}
+	if err := x.ClientStream.CloseSend(); err != nil {
+		return nil, err
+	}
+	return x, nil
+}
+
+type Store_ListClient interface {
+	Recv() (*ListResponse, error)
+	grpc.ClientStream
+}
+
+type storeListClient struct {
+	grpc.ClientStream
+}
+
+func (x *storeListClient) Recv() (*ListResponse, error) {
+	m := new(ListResponse)
+	if err := x.ClientStream.RecvMsg(m); err != nil {
+		return nil, err
+	}
+	return m, nil
+}
+
 func (c *storeClient) Read(ctx context.Context, in *ReadRequest, opts ...grpc.CallOption) (*ReadResponse, error) {
 	out := new(ReadResponse)
 	err := c.cc.Invoke(ctx, "/go.micro.store.Store/Read", in, out, opts...)
@@ -471,50 +503,39 @@ func (c *storeClient) Delete(ctx context.Context, in *DeleteRequest, opts ...grp
 	return out, nil
 }
 
-func (c *storeClient) Sync(ctx context.Context, in *SyncRequest, opts ...grpc.CallOption) (Store_SyncClient, error) {
-	stream, err := c.cc.NewStream(ctx, &_Store_serviceDesc.Streams[0], "/go.micro.store.Store/Sync", opts...)
-	if err != nil {
-		return nil, err
-	}
-	x := &storeSyncClient{stream}
-	if err := x.ClientStream.SendMsg(in); err != nil {
-		return nil, err
-	}
-	if err := x.ClientStream.CloseSend(); err != nil {
-		return nil, err
-	}
-	return x, nil
-}
-
-type Store_SyncClient interface {
-	Recv() (*SyncResponse, error)
-	grpc.ClientStream
-}
-
-type storeSyncClient struct {
-	grpc.ClientStream
-}
-
-func (x *storeSyncClient) Recv() (*SyncResponse, error) {
-	m := new(SyncResponse)
-	if err := x.ClientStream.RecvMsg(m); err != nil {
-		return nil, err
-	}
-	return m, nil
-}
-
 // StoreServer is the server API for Store service.
 type StoreServer interface {
+	List(*ListRequest, Store_ListServer) error
 	Read(context.Context, *ReadRequest) (*ReadResponse, error)
 	Write(context.Context, *WriteRequest) (*WriteResponse, error)
 	Delete(context.Context, *DeleteRequest) (*DeleteResponse, error)
-	Sync(*SyncRequest, Store_SyncServer) error
 }
 
 func RegisterStoreServer(s *grpc.Server, srv StoreServer) {
 	s.RegisterService(&_Store_serviceDesc, srv)
 }
 
+func _Store_List_Handler(srv interface{}, stream grpc.ServerStream) error {
+	m := new(ListRequest)
+	if err := stream.RecvMsg(m); err != nil {
+		return err
+	}
+	return srv.(StoreServer).List(m, &storeListServer{stream})
+}
+
+type Store_ListServer interface {
+	Send(*ListResponse) error
+	grpc.ServerStream
+}
+
+type storeListServer struct {
+	grpc.ServerStream
+}
+
+func (x *storeListServer) Send(m *ListResponse) error {
+	return x.ServerStream.SendMsg(m)
+}
+
 func _Store_Read_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) {
 	in := new(ReadRequest)
 	if err := dec(in); err != nil {
@@ -569,27 +590,6 @@ func _Store_Delete_Handler(srv interface{}, ctx context.Context, dec func(interf
 	return interceptor(ctx, in, info, handler)
 }
 
-func _Store_Sync_Handler(srv interface{}, stream grpc.ServerStream) error {
-	m := new(SyncRequest)
-	if err := stream.RecvMsg(m); err != nil {
-		return err
-	}
-	return srv.(StoreServer).Sync(m, &storeSyncServer{stream})
-}
-
-type Store_SyncServer interface {
-	Send(*SyncResponse) error
-	grpc.ServerStream
-}
-
-type storeSyncServer struct {
-	grpc.ServerStream
-}
-
-func (x *storeSyncServer) Send(m *SyncResponse) error {
-	return x.ServerStream.SendMsg(m)
-}
-
 var _Store_serviceDesc = grpc.ServiceDesc{
 	ServiceName: "go.micro.store.Store",
 	HandlerType: (*StoreServer)(nil),
@@ -609,8 +609,8 @@ var _Store_serviceDesc = grpc.ServiceDesc{
 	},
 	Streams: []grpc.StreamDesc{
 		{
-			StreamName:    "Sync",
-			Handler:       _Store_Sync_Handler,
+			StreamName:    "List",
+			Handler:       _Store_List_Handler,
 			ServerStreams: true,
 		},
 	},

store/service/proto/store.proto

@@ -3,10 +3,10 @@ syntax = "proto3";
 package go.micro.store;
 
 service Store {
+	rpc List(ListRequest) returns (stream ListResponse) {};
 	rpc Read(ReadRequest) returns (ReadResponse) {};
 	rpc Write(WriteRequest) returns (WriteResponse) {};
 	rpc Delete(DeleteRequest) returns (DeleteResponse) {};
-	rpc Sync(SyncRequest) returns (stream SyncResponse) {};
 }
 
 message Record {
@@ -38,11 +38,11 @@ message DeleteRequest {
 
 message DeleteResponse {}
 
-message SyncRequest {
+message ListRequest {
 	// optional key
 	string key = 1;
 }
 
-message SyncResponse {
+message ListResponse {
 	repeated Record records = 1;
 }

store/service/service.go

@@ -23,8 +23,8 @@ type serviceStore struct {
 }
 
 // Sync all the known records
-func (s *serviceStore) Sync() ([]*store.Record, error) {
-	stream, err := s.Client.Sync(context.Background(), &pb.SyncRequest{}, client.WithAddress(s.Nodes...))
+func (s *serviceStore) List() ([]*store.Record, error) {
+	stream, err := s.Client.List(context.Background(), &pb.ListRequest{}, client.WithAddress(s.Nodes...))
 	if err != nil {
 		return nil, err
 	}

store/store.go

@@ -4,26 +4,23 @@ package store
 import (
 	"errors"
 	"time"
-
-	"github.com/micro/go-micro/config/options"
 )
 
 var (
+	// ErrNotFound is returned when a Read key doesn't exist
 	ErrNotFound = errors.New("not found")
 )
 
 // Store is a data storage interface
 type Store interface {
-	// embed options
-	options.Options
-	// Sync all the known records
-	Sync() ([]*Record, error)
-	// Read a record with key
-	Read(keys ...string) ([]*Record, error)
-	// Write a record
-	Write(recs ...*Record) error
-	// Delete a record with key
-	Delete(keys ...string) error
+	// List all the known records
+	List() ([]*Record, error)
+	// Read records with keys
+	Read(key ...string) ([]*Record, error)
+	// Write records
+	Write(rec ...*Record) error
+	// Delete records with keys
+	Delete(key ...string) error
 }
 
 // Record represents a data record

sync/lock/memory/memory.go

@@ -1,4 +1,4 @@
-// Package memoy provides a sync.Mutex implementation of the lock for local use
+// Package memory provides a sync.Mutex implementation of the lock for local use
 package memory
 
 import (

sync/map.go

@@ -5,6 +5,7 @@ import (
 	"encoding/base64"
 	"encoding/json"
 	"fmt"
+	"sort"
 
 	"github.com/micro/go-micro/store"
 	ckv "github.com/micro/go-micro/store/etcd"
@@ -89,11 +90,15 @@ func (m *syncMap) Delete(key interface{}) error {
 }
 
 func (m *syncMap) Iterate(fn func(key, val interface{}) error) error {
-	keyvals, err := m.opts.Store.Sync()
+	keyvals, err := m.opts.Store.List()
 	if err != nil {
 		return err
 	}
 
+	sort.Slice(keyvals, func(i, j int) bool {
+		return keyvals[i].Key < keyvals[j].Key
+	})
+
 	for _, keyval := range keyvals {
 		// lock
 		if err := m.opts.Lock.Acquire(keyval.Key); err != nil {

sync/map_test.go

@@ -0,0 +1,39 @@
+package sync
+
+import (
+	"testing"
+	"time"
+
+	store "github.com/micro/go-micro/store"
+	mem_store "github.com/micro/go-micro/store/memory"
+	mem_lock "github.com/micro/go-micro/sync/lock/memory"
+)
+
+func TestIterate(t *testing.T) {
+	s1 := mem_store.NewStore()
+	s2 := mem_store.NewStore()
+	recA := &store.Record{
+		Key:   "A",
+		Value: nil,
+	}
+	recB := &store.Record{
+		Key:   "B",
+		Value: nil,
+	}
+	s1.Write(recA)
+	s1.Write(recB)
+	s2.Write(recB)
+	s2.Write(recA)
+
+	f := func(key, val interface{}) error {
+		time.Sleep(1 * time.Millisecond)
+		return nil
+	}
+	l := mem_lock.NewLock()
+	m1 := NewMap(WithStore(s1), WithLock(l))
+	m2 := NewMap(WithStore(s2), WithLock(l))
+	go func() {
+		m2.Iterate(f)
+	}()
+	m1.Iterate(f)
+}

tunnel/default.go

@@ -90,6 +90,7 @@ func (t *tun) getSession(channel, session string) (*session, bool) {
 	return s, ok
 }
 
+// delSession deletes a session if it exists
 func (t *tun) delSession(channel, session string) {
 	t.Lock()
 	delete(t.sessions, channel+session)
@@ -146,6 +147,9 @@ func (t *tun) newSessionId() string {
 	return uuid.New().String()
 }
 
+// announce will send a message to the link to tell the other side of a channel mapping we have.
+// This usually happens if someone calls Dial and sends a discover message but otherwise we
+// periodically send these messages to asynchronously manage channel mappings.
 func (t *tun) announce(channel, session string, link *link) {
 	// create the "announce" response message for a discover request
 	msg := &transport.Message{
@@ -206,7 +210,7 @@ func (t *tun) monitor() {
 			// check the link status and purge dead links
 			for node, link := range t.links {
 				// check link status
-				switch link.Status() {
+				switch link.State() {
 				case "closed":
 					delLinks = append(delLinks, node)
 				case "error":
@@ -221,9 +225,10 @@ func (t *tun) monitor() {
 				t.Lock()
 				for _, node := range delLinks {
 					log.Debugf("Tunnel deleting dead link for %s", node)
-					link := t.links[node]
-					link.Close()
-					delete(t.links, node)
+					if link, ok := t.links[node]; ok {
+						link.Close()
+						delete(t.links, node)
+					}
 				}
 				t.Unlock()
 			}
@@ -303,8 +308,16 @@ func (t *tun) process() {
 
 			// build the list of links ot send to
 			for node, link := range t.links {
+				// get the values we need
+				link.RLock()
+				id := link.id
+				connected := link.connected
+				loopback := link.loopback
+				_, exists := link.channels[msg.channel]
+				link.RUnlock()
+
 				// if the link is not connected skip it
-				if !link.connected {
+				if !connected {
 					log.Debugf("Link for node %s not connected", node)
 					err = errors.New("link not connected")
 					continue
@@ -313,32 +326,29 @@ func (t *tun) process() {
 				// if the link was a loopback accepted connection
 				// and the message is being sent outbound via
 				// a dialled connection don't use this link
-				if link.loopback && msg.outbound {
+				if loopback && msg.outbound {
 					err = errors.New("link is loopback")
 					continue
 				}
 
 				// if the message was being returned by the loopback listener
 				// send it back up the loopback link only
-				if msg.loopback && !link.loopback {
+				if msg.loopback && !loopback {
 					err = errors.New("link is not loopback")
 					continue
 				}
 
 				// check the multicast mappings
 				if msg.mode == Multicast {
-					link.RLock()
-					_, ok := link.channels[msg.channel]
-					link.RUnlock()
 					// channel mapping not found in link
-					if !ok {
+					if !exists {
 						continue
 					}
 				} else {
 					// if we're picking the link check the id
 					// this is where we explicitly set the link
 					// in a message received via the listen method
-					if len(msg.link) > 0 && link.id != msg.link {
+					if len(msg.link) > 0 && id != msg.link {
 						err = errors.New("link not found")
 						continue
 					}
@@ -353,7 +363,7 @@ func (t *tun) process() {
 			// send the message
 			for _, link := range sendTo {
 				// send the message via the current link
-				log.Debugf("Sending %+v to %s", newMsg.Header, link.Remote())
+				log.Tracef("Sending %+v to %s", newMsg.Header, link.Remote())
 
 				if errr := link.Send(newMsg); errr != nil {
 					log.Debugf("Tunnel error sending %+v to %s: %v", newMsg.Header, link.Remote(), errr)
@@ -422,12 +432,18 @@ func (t *tun) listen(link *link) {
 
 	// let us know if its a loopback
 	var loopback bool
+	var connected bool
+
+	// set the connected value
+	link.RLock()
+	connected = link.connected
+	link.RUnlock()
 
 	for {
 		// process anything via the net interface
 		msg := new(transport.Message)
 		if err := link.Recv(msg); err != nil {
-			log.Debugf("Tunnel link %s receive error: %#v", link.Remote(), err)
+			log.Debugf("Tunnel link %s receive error: %v", link.Remote(), err)
 			return
 		}
 
@@ -451,7 +467,7 @@ func (t *tun) listen(link *link) {
 
 		// if its not connected throw away the link
 		// the first message we process needs to be connect
-		if !link.connected && mtype != "connect" {
+		if !connected && mtype != "connect" {
 			log.Debugf("Tunnel link %s not connected", link.id)
 			return
 		}
@@ -461,7 +477,8 @@ func (t *tun) listen(link *link) {
 			log.Debugf("Tunnel link %s received connect message", link.Remote())
 
 			link.Lock()
-			// are we connecting to ourselves?
+
+			// check if we're connecting to ourselves?
 			if id == t.id {
 				link.loopback = true
 				loopback = true
@@ -471,6 +488,8 @@ func (t *tun) listen(link *link) {
 			link.id = link.Remote()
 			// set as connected
 			link.connected = true
+			connected = true
+
 			link.Unlock()
 
 			// save the link once connected
@@ -494,9 +513,7 @@ func (t *tun) listen(link *link) {
 
 			// the entire listener was closed so remove it from the mapping
 			if sessionId == "listener" {
-				link.Lock()
-				delete(link.channels, channel)
-				link.Unlock()
+				link.delChannel(channel)
 				continue
 			}
 
@@ -510,10 +527,8 @@ func (t *tun) listen(link *link) {
 			// otherwise its a session mapping of sorts
 		case "keepalive":
 			log.Debugf("Tunnel link %s received keepalive", link.Remote())
-			link.Lock()
 			// save the keepalive
-			link.lastKeepAlive = time.Now()
-			link.Unlock()
+			link.keepalive()
 			continue
 		// a new connection dialled outbound
 		case "open":
@@ -533,18 +548,14 @@ func (t *tun) listen(link *link) {
 		// a continued session
 		case "session":
 			// process message
-			log.Debugf("Received %+v from %s", msg.Header, link.Remote())
+			log.Tracef("Received %+v from %s", msg.Header, link.Remote())
 		// an announcement of a channel listener
 		case "announce":
 			// process the announcement
 			channels := strings.Split(channel, ",")
 
 			// update mapping in the link
-			link.Lock()
-			for _, channel := range channels {
-				link.channels[channel] = time.Now()
-			}
-			link.Unlock()
+			link.setChannel(channels...)
 
 			// this was an announcement not intended for anything
 			if sessionId == "listener" || sessionId == "" {
@@ -579,7 +590,7 @@ func (t *tun) listen(link *link) {
 		}
 
 		// strip tunnel message header
-		for k, _ := range msg.Header {
+		for k := range msg.Header {
 			if strings.HasPrefix(k, "Micro-Tunnel") {
 				delete(msg.Header, k)
 			}
@@ -746,8 +757,13 @@ func (t *tun) setupLink(node string) (*link, error) {
 	}
 	log.Debugf("Tunnel connected to %s", node)
 
+	// create a new link
+	link := newLink(c)
+	// set link id to remote side
+	link.id = c.Remote()
+
 	// send the first connect message
-	if err := c.Send(&transport.Message{
+	if err := link.Send(&transport.Message{
 		Header: map[string]string{
 			"Micro-Tunnel":       "connect",
 			"Micro-Tunnel-Id":    t.id,
@@ -757,10 +773,6 @@ func (t *tun) setupLink(node string) (*link, error) {
 		return nil, err
 	}
 
-	// create a new link
-	link := newLink(c)
-	// set link id to remote side
-	link.id = c.Remote()
 	// we made the outbound connection
 	// and sent the connect message
 	link.connected = true
@@ -903,6 +915,53 @@ func (t *tun) close() error {
 	return t.listener.Close()
 }
 
+// pickLink will pick the best link based on connectivity, delay, rate and length
+func (t *tun) pickLink(links []*link) *link {
+	var metric float64
+	var chosen *link
+
+	// find the best link
+	for i, link := range links {
+		// don't use disconnected or errored links
+		if link.State() != "connected" {
+			continue
+		}
+
+		// get the link state info
+		d := float64(link.Delay())
+		l := float64(link.Length())
+		r := link.Rate()
+
+		// metric = delay x length x rate
+		m := d * l * r
+
+		// first link so just and go
+		if i == 0 {
+			metric = m
+			chosen = link
+			continue
+		}
+
+		// we found a better metric
+		if m < metric {
+			metric = m
+			chosen = link
+		}
+	}
+
+	// if there's no link we're just going to mess around
+	if chosen == nil {
+		i := rand.Intn(len(links))
+		return links[i]
+	}
+
+	// we chose the link with;
+	// the lowest delay e.g least messages queued
+	// the lowest rate e.g the least messages flowing
+	// the lowest length e.g the smallest roundtrip time
+	return chosen
+}
+
 func (t *tun) Address() string {
 	t.RLock()
 	defer t.RUnlock()
@@ -967,128 +1026,106 @@ func (t *tun) Dial(channel string, opts ...DialOption) (Session, error) {
 	// set the dial timeout
 	c.timeout = options.Timeout
 
-	now := time.Now()
+	var links []*link
+	// did we measure the rtt
+	var measured bool
 
-	after := func() time.Duration {
-		d := time.Since(now)
-		// dial timeout minus time since
-		wait := options.Timeout - d
-		if wait < time.Duration(0) {
-			return time.Duration(0)
-		}
-		return wait
-	}
-
-	var links []string
+	t.RLock()
 
 	// non multicast so we need to find the link
-	if id := options.Link; id != "" {
-		t.RLock()
-		for _, link := range t.links {
-			// use the link specified it its available
-			if link.id != id {
-				continue
-			}
-
-			link.RLock()
-			_, ok := link.channels[channel]
-			link.RUnlock()
-
-			// we have at least one channel mapping
-			if ok {
-				c.discovered = true
-				links = append(links, link.id)
-			}
-		}
-		t.RUnlock()
-		// link not found
-		if len(links) == 0 {
-			// delete session and return error
-			t.delSession(c.channel, c.session)
-			return nil, ErrLinkNotFound
+	for _, link := range t.links {
+		// use the link specified it its available
+		if id := options.Link; len(id) > 0 && link.id != id {
+			continue
 		}
 
+		// get the channel
+		lastMapped := link.getChannel(channel)
+
+		// we have at least one channel mapping
+		if !lastMapped.IsZero() {
+			links = append(links, link)
+			c.discovered = true
+		}
+	}
+
+	t.RUnlock()
+
+	// link not found
+	if len(links) == 0 && len(options.Link) > 0 {
+		// delete session and return error
+		t.delSession(c.channel, c.session)
+		log.Debugf("Tunnel deleting session %s %s: %v", c.session, c.channel, ErrLinkNotFound)
+		return nil, ErrLinkNotFound
 	}
 
 	// discovered so set the link if not multicast
 	// TODO: pick the link efficiently based
 	// on link status and saturation.
 	if c.discovered && c.mode == Unicast {
-		// set the link
-		i := rand.Intn(len(links))
-		c.link = links[i]
+		// pickLink will pick the best link
+		link := t.pickLink(links)
+		c.link = link.id
 	}
 
 	// shit fuck
 	if !c.discovered {
-		// create a new discovery message for this channel
-		msg := c.newMessage("discover")
-		msg.mode = Broadcast
-		msg.outbound = true
-		msg.link = ""
+		// piggy back roundtrip
+		nowRTT := time.Now()
 
-		// send the discovery message
-		t.send <- msg
-
-		select {
-		case <-time.After(after()):
-			t.delSession(c.channel, c.session)
-			return nil, ErrDialTimeout
-		case err := <-c.errChan:
-			if err != nil {
-				t.delSession(c.channel, c.session)
-				return nil, err
-			}
-		}
-
-		var err error
-
-		// set a dialTimeout
-		dialTimeout := after()
-
-		// set a shorter delay for multicast
-		if c.mode != Unicast {
-			// shorten this
-			dialTimeout = time.Millisecond * 500
-		}
-
-		// wait for announce
-		select {
-		case msg := <-c.recv:
-			if msg.typ != "announce" {
-				err = ErrDiscoverChan
-			}
-		case <-time.After(dialTimeout):
-			err = ErrDialTimeout
-		}
-
-		// if its multicast just go ahead because this is best effort
-		if c.mode != Unicast {
-			c.discovered = true
-			c.accepted = true
-			return c, nil
-		}
-
-		// otherwise return an error
+		// attempt to discover the link
+		err := c.Discover()
 		if err != nil {
 			t.delSession(c.channel, c.session)
+			log.Debugf("Tunnel deleting session %s %s: %v", c.session, c.channel, err)
 			return nil, err
 		}
 
-		// set discovered to true
-		c.discovered = true
+		// set roundtrip
+		d := time.Since(nowRTT)
+
+		// set the link time
+		t.RLock()
+		link, ok := t.links[c.link]
+		t.RUnlock()
+
+		if ok {
+			// set the rountrip time
+			link.setRTT(d)
+			// set measured to true
+			measured = true
+		}
 	}
 
 	// a unicast session so we call "open" and wait for an "accept"
 
+	// reset now in case we use it
+	now := time.Now()
+
 	// try to open the session
-	err := c.Open()
-	if err != nil {
+	if err := c.Open(); err != nil {
 		// delete the session
 		t.delSession(c.channel, c.session)
+		log.Debugf("Tunnel deleting session %s %s: %v", c.session, c.channel, err)
 		return nil, err
 	}
 
+	// set time take to open
+	d := time.Since(now)
+
+	// if we haven't measured the roundtrip do it now
+	if !measured && c.mode == Unicast {
+		// set the link time
+		t.RLock()
+		link, ok := t.links[c.link]
+		t.RUnlock()
+
+		if ok {
+			// set the rountrip time
+			link.setRTT(d)
+		}
+	}
+
 	return c, nil
 }
 

tunnel/link.go

@@ -1,11 +1,14 @@
 package tunnel
 
 import (
+	"bytes"
+	"io"
 	"sync"
 	"time"
 
 	"github.com/google/uuid"
 	"github.com/micro/go-micro/transport"
+	"github.com/micro/go-micro/util/log"
 )
 
 type link struct {
@@ -14,6 +17,12 @@ type link struct {
 	sync.RWMutex
 	// stops the link
 	closed chan bool
+	// link state channel for testing link
+	state chan *packet
+	// send queue for sending packets
+	sendQueue chan *packet
+	// receive queue for receiving packets
+	recvQueue chan *packet
 	// unique id of this link e.g uuid
 	// which we define for ourselves
 	id string
@@ -31,32 +40,208 @@ type link struct {
 	lastKeepAlive time.Time
 	// channels keeps a mapping of channels and last seen
 	channels map[string]time.Time
-
+	// the weighted moving average roundtrip
+	length int64
+	// weighted moving average of bits flowing
+	rate float64
 	// keep an error count on the link
 	errCount int
 }
 
+// packet send over link
+type packet struct {
+	// message to send or received
+	message *transport.Message
+
+	// status returned when sent
+	status chan error
+
+	// receive related error
+	err error
+}
+
+var (
+	// the 4 byte 0 packet sent to determine the link state
+	linkRequest = []byte{0, 0, 0, 0}
+	// the 4 byte 1 filled packet sent to determine link state
+	linkResponse = []byte{1, 1, 1, 1}
+)
+
 func newLink(s transport.Socket) *link {
 	l := &link{
 		Socket:        s,
 		id:            uuid.New().String(),
+		lastKeepAlive: time.Now(),
 		channels:      make(map[string]time.Time),
 		closed:        make(chan bool),
-		lastKeepAlive: time.Now(),
+		state:         make(chan *packet, 64),
+		sendQueue:     make(chan *packet, 128),
+		recvQueue:     make(chan *packet, 128),
 	}
-	go l.expiry()
+
+	// process inbound/outbound packets
+	go l.process()
+	// manage the link state
+	go l.manage()
+
 	return l
 }
 
-// watches the channel expiry
-func (l *link) expiry() {
-	t := time.NewTicker(time.Minute)
-	defer t.Stop()
+// setRate sets the bits per second rate as a float64
+func (l *link) setRate(bits int64, delta time.Duration) {
+	// rate of send in bits per nanosecond
+	rate := float64(bits) / float64(delta.Nanoseconds())
+
+	// default the rate if its zero
+	if l.rate == 0 {
+		// rate per second
+		l.rate = rate * 1e9
+	} else {
+		// set new rate per second
+		l.rate = 0.8*l.rate + 0.2*(rate*1e9)
+	}
+}
+
+// setRTT sets a nanosecond based moving average roundtrip time for the link
+func (l *link) setRTT(d time.Duration) {
+	l.Lock()
+	defer l.Unlock()
+
+	if l.length <= 0 {
+		l.length = d.Nanoseconds()
+		return
+	}
+
+	// https://fishi.devtail.io/weblog/2015/04/12/measuring-bandwidth-and-round-trip-time-tcp-connection-inside-application-layer/
+	length := 0.8*float64(l.length) + 0.2*float64(d.Nanoseconds())
+	// set new length
+	l.length = int64(length)
+}
+
+func (l *link) delChannel(ch string) {
+	l.Lock()
+	delete(l.channels, ch)
+	l.Unlock()
+}
+
+func (l *link) getChannel(ch string) time.Time {
+	l.RLock()
+	defer l.RUnlock()
+	return l.channels[ch]
+}
+
+func (l *link) setChannel(channels ...string) {
+	l.Lock()
+	for _, ch := range channels {
+		l.channels[ch] = time.Now()
+	}
+	l.Unlock()
+}
+
+// set the keepalive time
+func (l *link) keepalive() {
+	l.Lock()
+	l.lastKeepAlive = time.Now()
+	l.Unlock()
+}
+
+// process deals with the send queue
+func (l *link) process() {
+	// receive messages
+	go func() {
+		for {
+			m := new(transport.Message)
+			err := l.recv(m)
+			if err != nil {
+				l.Lock()
+				l.errCount++
+				l.Unlock()
+			}
+
+			// process new received message
+
+			pk := &packet{message: m, err: err}
+
+			// this is our link state packet
+			if m.Header["Micro-Method"] == "link" {
+				// process link state message
+				select {
+				case l.state <- pk:
+				default:
+				}
+				continue
+			}
+
+			// process all messages as is
+
+			select {
+			case l.recvQueue <- pk:
+			case <-l.closed:
+				return
+			}
+		}
+	}()
+
+	// send messages
 
 	for {
 		select {
+		case pk := <-l.sendQueue:
+			// send the message
+			pk.status <- l.send(pk.message)
 		case <-l.closed:
 			return
+		}
+	}
+}
+
+// manage manages the link state including rtt packets and channel mapping expiry
+func (l *link) manage() {
+	// tick over every minute to expire and fire rtt packets
+	t := time.NewTicker(time.Minute)
+	defer t.Stop()
+
+	// used to send link state packets
+	send := func(b []byte) error {
+		return l.Send(&transport.Message{
+			Header: map[string]string{
+				"Micro-Method": "link",
+			}, Body: b,
+		})
+	}
+
+	// set time now
+	now := time.Now()
+
+	// send the initial rtt request packet
+	send(linkRequest)
+
+	for {
+		select {
+		// exit if closed
+		case <-l.closed:
+			return
+		// process link state rtt packets
+		case p := <-l.state:
+			if p.err != nil {
+				continue
+			}
+			// check the type of message
+			switch {
+			case bytes.Compare(p.message.Body, linkRequest) == 0:
+				log.Tracef("Link %s received link request %v", l.id, p.message.Body)
+				// send response
+				if err := send(linkResponse); err != nil {
+					l.Lock()
+					l.errCount++
+					l.Unlock()
+				}
+			case bytes.Compare(p.message.Body, linkResponse) == 0:
+				// set round trip time
+				d := time.Since(now)
+				log.Tracef("Link %s received link response in %v", p.message.Body, d)
+				l.setRTT(d)
+			}
 		case <-t.C:
 			// drop any channel mappings older than 2 minutes
 			var kill []string
@@ -81,10 +266,51 @@ func (l *link) expiry() {
 				delete(l.channels, ch)
 			}
 			l.Unlock()
+
+			// fire off a link state rtt packet
+			now = time.Now()
+			send(linkRequest)
 		}
 	}
 }
 
+func (l *link) send(m *transport.Message) error {
+	if m.Header == nil {
+		m.Header = make(map[string]string)
+	}
+	// send the message
+	return l.Socket.Send(m)
+}
+
+// recv a message on the link
+func (l *link) recv(m *transport.Message) error {
+	if m.Header == nil {
+		m.Header = make(map[string]string)
+	}
+	// receive the transport message
+	return l.Socket.Recv(m)
+}
+
+// Delay is the current load on the link
+func (l *link) Delay() int64 {
+	return int64(len(l.sendQueue) + len(l.recvQueue))
+}
+
+// Current transfer rate as bits per second (lower is better)
+func (l *link) Rate() float64 {
+	l.RLock()
+	defer l.RUnlock()
+	return l.rate
+}
+
+// Length returns the roundtrip time as nanoseconds (lower is better).
+// Returns 0 where no measurement has been taken.
+func (l *link) Length() int64 {
+	l.RLock()
+	defer l.RUnlock()
+	return l.length
+}
+
 func (l *link) Id() string {
 	l.RLock()
 	defer l.RUnlock()
@@ -93,34 +319,115 @@ func (l *link) Id() string {
 }
 
 func (l *link) Close() error {
+	l.Lock()
+	defer l.Unlock()
+
 	select {
 	case <-l.closed:
 		return nil
 	default:
+		l.Socket.Close()
 		close(l.closed)
 	}
 
 	return nil
 }
 
+// Send sencs a message on the link
 func (l *link) Send(m *transport.Message) error {
-	err := l.Socket.Send(m)
+	// create a new packet to send over the link
+	p := &packet{
+		message: m,
+		status:  make(chan error, 1),
+	}
+
+	// get time now
+	now := time.Now()
+
+	// check if its closed first
+	select {
+	case <-l.closed:
+		return io.EOF
+	default:
+	}
+
+	// queue the message
+	select {
+	case l.sendQueue <- p:
+		// in the send queue
+	case <-l.closed:
+		return io.EOF
+	}
+
+	// error to use
+	var err error
+
+	// wait for response
+	select {
+	case <-l.closed:
+		return io.EOF
+	case err = <-p.status:
+	}
 
 	l.Lock()
 	defer l.Unlock()
 
-	// if theres no error reset the counter
-	if err == nil {
-		l.errCount = 0
+	// there's an error increment the counter and bail
+	if err != nil {
+		l.errCount++
+		return err
 	}
 
-	// otherwise increment the counter
-	l.errCount++
+	// reset the counter
+	l.errCount = 0
 
-	return err
+	// calculate the data sent
+	dataSent := len(m.Body)
+
+	// set header length
+	for k, v := range m.Header {
+		dataSent += (len(k) + len(v))
+	}
+
+	// calculate based on data
+	if dataSent > 0 {
+		// bit sent
+		bits := dataSent * 1024
+
+		// set the rate
+		l.setRate(int64(bits), time.Since(now))
+	}
+
+	return nil
 }
 
-func (l *link) Status() string {
+// Accept accepts a message on the socket
+func (l *link) Recv(m *transport.Message) error {
+	select {
+	case <-l.closed:
+		// check if there's any messages left
+		select {
+		case pk := <-l.recvQueue:
+			// check the packet receive error
+			if pk.err != nil {
+				return pk.err
+			}
+			*m = *pk.message
+		default:
+			return io.EOF
+		}
+	case pk := <-l.recvQueue:
+		// check the packet receive error
+		if pk.err != nil {
+			return pk.err
+		}
+		*m = *pk.message
+	}
+	return nil
+}
+
+// State can return connected, closed, error
+func (l *link) State() string {
 	select {
 	case <-l.closed:
 		return "closed"

tunnel/session.go

@@ -106,6 +106,106 @@ func (s *session) newMessage(typ string) *message {
 	}
 }
 
+// waitFor waits for the message type required until the timeout specified
+func (s *session) waitFor(msgType string, timeout time.Duration) (*message, error) {
+	now := time.Now()
+
+	after := func(timeout time.Duration) time.Duration {
+		d := time.Since(now)
+		// dial timeout minus time since
+		wait := timeout - d
+
+		if wait < time.Duration(0) {
+			return time.Duration(0)
+		}
+
+		return wait
+	}
+
+	// wait for the message type
+	for {
+		select {
+		case msg := <-s.recv:
+			// ignore what we don't want
+			if msg.typ != msgType {
+				log.Debugf("Tunnel received non %s message in waiting for %s", msg.typ, msgType)
+				continue
+			}
+			// got the message
+			return msg, nil
+		case <-time.After(after(timeout)):
+			return nil, ErrDialTimeout
+		case <-s.closed:
+			return nil, io.EOF
+		}
+	}
+}
+
+// Discover attempts to discover the link for a specific channel
+func (s *session) Discover() error {
+	// create a new discovery message for this channel
+	msg := s.newMessage("discover")
+	msg.mode = Broadcast
+	msg.outbound = true
+	msg.link = ""
+
+	// send the discovery message
+	s.send <- msg
+
+	// set time now
+	now := time.Now()
+
+	after := func() time.Duration {
+		d := time.Since(now)
+		// dial timeout minus time since
+		wait := s.timeout - d
+		if wait < time.Duration(0) {
+			return time.Duration(0)
+		}
+		return wait
+	}
+
+	// wait to hear back about the sent message
+	select {
+	case <-time.After(after()):
+		return ErrDialTimeout
+	case err := <-s.errChan:
+		if err != nil {
+			return err
+		}
+	}
+
+	var err error
+
+	// set a new dialTimeout
+	dialTimeout := after()
+
+	// set a shorter delay for multicast
+	if s.mode != Unicast {
+		// shorten this
+		dialTimeout = time.Millisecond * 500
+	}
+
+	// wait for announce
+	_, err = s.waitFor("announce", dialTimeout)
+
+	// if its multicast just go ahead because this is best effort
+	if s.mode != Unicast {
+		s.discovered = true
+		s.accepted = true
+		return nil
+	}
+
+	if err != nil {
+		return err
+	}
+
+	// set discovered
+	s.discovered = true
+
+	return nil
+}
+
 // Open will fire the open message for the session. This is called by the dialler.
 func (s *session) Open() error {
 	// create a new message
@@ -131,22 +231,16 @@ func (s *session) Open() error {
 	}
 
 	// now wait for the accept
-	select {
-	case msg = <-s.recv:
-		if msg.typ != "accept" {
-			log.Debugf("Received non accept message in Open %s", msg.typ)
-			return errors.New("failed to connect")
-		}
-		// set to accepted
-		s.accepted = true
-		// set link
-		s.link = msg.link
-	case <-time.After(s.timeout):
-		return ErrDialTimeout
-	case <-s.closed:
-		return io.EOF
+	msg, err := s.waitFor("accept", s.timeout)
+	if err != nil {
+		return err
 	}
 
+	// set to accepted
+	s.accepted = true
+	// set link
+	s.link = msg.link
+
 	return nil
 }
 
@@ -227,7 +321,7 @@ func (s *session) Send(m *transport.Message) error {
 		msg.link = ""
 	}
 
-	log.Debugf("Appending %+v to send backlog", msg)
+	log.Tracef("Appending %+v to send backlog", msg)
 	// send the actual message
 	s.send <- msg
 
@@ -258,7 +352,7 @@ func (s *session) Recv(m *transport.Message) error {
 	default:
 	}
 
-	log.Debugf("Received %+v from recv backlog", msg)
+	log.Tracef("Received %+v from recv backlog", msg)
 	// set message
 	*m = *msg.data
 	// return nil

tunnel/tunnel.go

@@ -57,8 +57,14 @@ type Tunnel interface {
 type Link interface {
 	// The id of the link
 	Id() string
-	// Status of the link e.g connected/closed
-	Status() string
+	// Delay is the current load on the link (lower is better)
+	Delay() int64
+	// Length returns the roundtrip time as nanoseconds (lower is better)
+	Length() int64
+	// Current transfer rate as bits per second (lower is better)
+	Rate() float64
+	// State of the link e.g connected/closed
+	State() string
 	// honours transport socket
 	transport.Socket
 }

tunnel/tunnel_test.go

@@ -202,8 +202,8 @@ func testBrokenTunAccept(t *testing.T, tun Tunnel, wait chan bool, wg *sync.Wait
 		t.Fatal(err)
 	}
 
-	// notify sender we have received the message
-	<-wait
+	// notify the sender we have received
+	wait <- true
 }
 
 func testBrokenTunSend(t *testing.T, tun Tunnel, wait chan bool, wg *sync.WaitGroup) {
@@ -234,7 +234,7 @@ func testBrokenTunSend(t *testing.T, tun Tunnel, wait chan bool, wg *sync.WaitGr
 	<-wait
 
 	// give it time to reconnect
-	time.Sleep(2 * ReconnectTime)
+	time.Sleep(5 * ReconnectTime)
 
 	// send the message
 	if err := c.Send(&m); err != nil {
@@ -244,7 +244,7 @@ func testBrokenTunSend(t *testing.T, tun Tunnel, wait chan bool, wg *sync.WaitGr
 	// wait for the listener to receive the message
 	// c.Send merely enqueues the message to the link send queue and returns
 	// in order to verify it was received we wait for the listener to tell us
-	wait <- true
+	<-wait
 }
 
 func TestReconnectTunnel(t *testing.T) {
@@ -292,3 +292,53 @@ func TestReconnectTunnel(t *testing.T) {
 	// wait until done
 	wg.Wait()
 }
+
+func TestTunnelRTTRate(t *testing.T) {
+	// create a new tunnel client
+	tunA := NewTunnel(
+		Address("127.0.0.1:9096"),
+		Nodes("127.0.0.1:9097"),
+	)
+
+	// create a new tunnel server
+	tunB := NewTunnel(
+		Address("127.0.0.1:9097"),
+	)
+
+	// start tunB
+	err := tunB.Connect()
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer tunB.Close()
+
+	// start tunA
+	err = tunA.Connect()
+	if err != nil {
+		t.Fatal(err)
+	}
+	defer tunA.Close()
+
+	wait := make(chan bool)
+
+	var wg sync.WaitGroup
+
+	wg.Add(1)
+	// start the listener
+	go testAccept(t, tunB, wait, &wg)
+
+	wg.Add(1)
+	// start the client
+	go testSend(t, tunA, wait, &wg)
+
+	// wait until done
+	wg.Wait()
+
+	for _, link := range tunA.Links() {
+		t.Logf("Link %s length %v rate %v", link.Id(), link.Length(), link.Rate())
+	}
+
+	for _, link := range tunB.Links() {
+		t.Logf("Link %s length %v rate %v", link.Id(), link.Length(), link.Rate())
+	}
+}

web/service.go

@@ -82,7 +82,7 @@ func (s *service) genSrv() *registry.Service {
 	return &registry.Service{
 		Name:    s.opts.Name,
 		Version: s.opts.Version,
-		Nodes: []*registry.Node{&registry.Node{
+		Nodes: []*registry.Node{{
 			Id:       s.opts.Id,
 			Address:  fmt.Sprintf("%s:%d", addr, port),
 			Metadata: s.opts.Metadata,
@@ -118,6 +118,11 @@ func (s *service) register() error {
 	if s.opts.Registry != nil {
 		r = s.opts.Registry
 	}
+
+	// service node need modify, node address maybe changed
+	srv := s.genSrv()
+	srv.Endpoints = s.srv.Endpoints
+	s.srv = srv
 	return r.Register(s.srv, registry.RegisterTTL(s.opts.RegisterTTL))
 }