gofmt

Update marshaler.go of codec/proto.

.travis.yml

@@ -2,6 +2,8 @@ language: go
 go:
 - 1.11.x
 - 1.12.x
+env:
+  - GO111MODULE=on
 notifications:
   slack:
     secure: aEvhLbhujaGaKSrOokiG3//PaVHTIrc3fBpoRbCRqfZpyq6WREoapJJhF+tIpWWOwaC9GmChbD6aHo/jMUgwKXVyPSaNjiEL87YzUUpL8B2zslNp1rgfTg/LrzthOx3Q1TYwpaAl3to0fuHUVFX4yMeC2vuThq7WSXgMMxFCtbc=

README.md

@@ -1,6 +1,6 @@
 # Go Micro [![License](https://img.shields.io/:license-apache-blue.svg)](https://opensource.org/licenses/Apache-2.0) [![GoDoc](https://godoc.org/github.com/micro/go-micro?status.svg)](https://godoc.org/github.com/micro/go-micro) [![Travis CI](https://api.travis-ci.org/micro/go-micro.svg?branch=master)](https://travis-ci.org/micro/go-micro) [![Go Report Card](https://goreportcard.com/badge/micro/go-micro)](https://goreportcard.com/report/github.com/micro/go-micro)
 
-Go Micro is a framework for micro service development.
+Go Micro is a framework for microservice development.
 
 ## Overview
 
@@ -47,8 +47,3 @@ are pluggable and allows Go Micro to be runtime agnostic. You can plugin any und
 
 See the [docs](https://micro.mu/docs/go-micro.html) for detailed information on the architecture, installation and use of go-micro.
 
-## Sponsors
-
-Sixt is an Enterprise Sponsor of Micro
-
-<a href="https://micro.mu/blog/2016/04/25/announcing-sixt-sponsorship.html"><img src="https://micro.mu/sixt_logo.png" width=150px height="auto" /></a>

README.zh-cn.md

@@ -34,8 +34,3 @@ Go Micro把分布式系统的各种细节抽象出来。下面是它的主要特
 
 更多关于架构、安装的资料可以查看[文档](https://micro.mu/docs/go-micro_cn.html)。
 
-## 赞助商
-
-Sixt是我们的赞助商。
-
-<a href="https://micro.mu/blog/2016/04/25/announcing-sixt-sponsorship.html"><img src="https://micro.mu/sixt_logo.png" width=150px height="auto" /></a>

agent/README.md

@@ -0,0 +1,197 @@
+# Agent 
+
+Agent is a library used to create commands, inputs and robot services
+
+## Getting Started
+
+- [Commands](#commands) - Commands are functions executed by the bot based on text based pattern matching.
+- [Inputs](#inputs) - Inputs are plugins for communication e.g Slack, Telegram, IRC, etc.
+- [Services](#services) - Write bots as micro services
+
+## Commands
+
+Commands are functions executed by the bot based on text based pattern matching.
+
+### Write a Command
+
+```go
+import "github.com/micro/go-micro/agent/command"
+
+func Ping() command.Command {
+	usage := "ping"
+	description := "Returns pong"
+
+	return command.NewCommand("ping", usage, desc, func(args ...string) ([]byte, error) {
+		return []byte("pong"), nil
+	})
+}
+```
+
+### Register the command
+
+Add the command to the Commands map with a pattern key that can be matched by golang/regexp.Match
+
+```go
+import "github.com/micro/go-micro/agent/command"
+
+func init() {
+	command.Commands["^ping$"] = Ping()
+}
+```
+
+### Rebuild Micro
+
+Build binary
+```shell
+cd github.com/micro/micro
+
+// For local use
+go build -i -o micro ./main.go
+
+// For docker image
+CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-w' -i -o micro ./main.go
+```
+
+## Inputs
+
+Inputs are plugins for communication e.g Slack, HipChat, XMPP, IRC, SMTP, etc, etc. 
+
+New inputs can be added in the following way.
+
+### Write an Input
+
+Write an input that satisfies the Input interface.
+
+```go
+type Input interface {
+	// Provide cli flags
+	Flags() []cli.Flag
+	// Initialise input using cli context
+	Init(*cli.Context) error
+	// Stream events from the input
+	Stream() (Conn, error)
+	// Start the input
+	Start() error
+	// Stop the input
+	Stop() error
+	// name of the input
+	String() string
+}
+```
+
+### Register the input
+
+Add the input to the Inputs map.
+
+```go
+import "github.com/micro/micro/bot/input"
+
+func init() {
+	input.Inputs["name"] = MyInput
+}
+```
+
+### Rebuild Micro
+
+Build binary
+```shell
+cd github.com/micro/micro
+
+// For local use
+go build -i -o micro ./main.go
+
+// For docker image
+CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-w' -i -o micro ./main.go
+```
+
+## Services
+
+The micro bot supports the ability to create commands as micro services. 
+
+### How does it work?
+
+The bot watches the service registry for services with it's namespace. The default namespace is `go.micro.bot`. 
+Any service within this namespace will automatically be added to the list of available commands. When a command 
+is executed, the bot will call the service with method `Command.Exec`. It also expects the method `Command.Help` 
+to exist for usage info.
+
+
+The service interface is as follows and can be found at [go-micro/agent/proto](https://github.com/micro/go-micro/agent/blob/master/proto/bot.proto)
+
+```
+syntax = "proto3";
+
+package go.micro.bot;
+
+service Command {
+	rpc Help(HelpRequest) returns (HelpResponse) {};
+	rpc Exec(ExecRequest) returns (ExecResponse) {};
+}
+
+message HelpRequest {
+}
+
+message HelpResponse {
+	string usage = 1;
+	string description = 2;
+}
+
+message ExecRequest {
+	repeated string args = 1;
+}
+
+message ExecResponse {
+	bytes result = 1;
+	string error = 2;
+}
+```
+
+### Example
+
+Here's an example echo command as a microservice
+
+```go
+package main
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/micro/go-micro"
+	"golang.org/x/net/context"
+
+	proto "github.com/micro/go-micro/agent/proto"
+)
+
+type Command struct{}
+
+// Help returns the command usage
+func (c *Command) Help(ctx context.Context, req *proto.HelpRequest, rsp *proto.HelpResponse) error {
+	// Usage should include the name of the command
+	rsp.Usage = "echo"
+	rsp.Description = "This is an example bot command as a micro service which echos the message"
+	return nil
+}
+
+// Exec executes the command
+func (c *Command) Exec(ctx context.Context, req *proto.ExecRequest, rsp *proto.ExecResponse) error {
+	rsp.Result = []byte(strings.Join(req.Args, " "))
+	// rsp.Error could be set to return an error instead
+	// the function error would only be used for service level issues
+	return nil
+}
+
+func main() {
+	service := micro.NewService(
+		micro.Name("go.micro.bot.echo"),
+	)
+
+	service.Init()
+
+	proto.RegisterCommandHandler(service.Server(), new(Command))
+
+	if err := service.Run(); err != nil {
+		fmt.Println(err)
+	}
+}
+```

agent/agent.go

@@ -0,0 +1,2 @@
+// Package agent provides an interface for building robots
+package agent

agent/command/command.go

@@ -0,0 +1,54 @@
+// Package command is an interface for defining bot commands
+package command
+
+var (
+	// Commmands keyed by golang/regexp patterns
+	// regexp.Match(key, input) is used to match
+	Commands = map[string]Command{}
+)
+
+// Command is the interface for specific named
+// commands executed via plugins or the bot.
+type Command interface {
+	// Executes the command with args passed in
+	Exec(args ...string) ([]byte, error)
+	// Usage of the command
+	Usage() string
+	// Description of the command
+	Description() string
+	// Name of the command
+	String() string
+}
+
+type cmd struct {
+	name        string
+	usage       string
+	description string
+	exec        func(args ...string) ([]byte, error)
+}
+
+func (c *cmd) Description() string {
+	return c.description
+}
+
+func (c *cmd) Exec(args ...string) ([]byte, error) {
+	return c.exec(args...)
+}
+
+func (c *cmd) Usage() string {
+	return c.usage
+}
+
+func (c *cmd) String() string {
+	return c.name
+}
+
+// NewCommand helps quickly create a new command
+func NewCommand(name, usage, description string, exec func(args ...string) ([]byte, error)) Command {
+	return &cmd{
+		name:        name,
+		usage:       usage,
+		description: description,
+		exec:        exec,
+	}
+}

agent/command/command_test.go

@@ -0,0 +1,65 @@
+package command
+
+import (
+	"testing"
+)
+
+func TestCommand(t *testing.T) {
+	c := &cmd{
+		name:        "test",
+		usage:       "test usage",
+		description: "test description",
+		exec: func(args ...string) ([]byte, error) {
+			return []byte("test"), nil
+		},
+	}
+
+	if c.String() != c.name {
+		t.Fatalf("expected name %s got %s", c.name, c.String())
+	}
+
+	if c.Usage() != c.usage {
+		t.Fatalf("expected usage %s got %s", c.usage, c.Usage())
+	}
+
+	if c.Description() != c.description {
+		t.Fatalf("expected description %s got %s", c.description, c.Description())
+	}
+
+	if r, err := c.Exec(); err != nil {
+		t.Fatal(err)
+	} else if string(r) != "test" {
+		t.Fatalf("expected exec result test got %s", string(r))
+	}
+}
+
+func TestNewCommand(t *testing.T) {
+	c := &cmd{
+		name:        "test",
+		usage:       "test usage",
+		description: "test description",
+		exec: func(args ...string) ([]byte, error) {
+			return []byte("test"), nil
+		},
+	}
+
+	nc := NewCommand(c.name, c.usage, c.description, c.exec)
+
+	if nc.String() != c.name {
+		t.Fatalf("expected name %s got %s", c.name, nc.String())
+	}
+
+	if nc.Usage() != c.usage {
+		t.Fatalf("expected usage %s got %s", c.usage, nc.Usage())
+	}
+
+	if nc.Description() != c.description {
+		t.Fatalf("expected description %s got %s", c.description, nc.Description())
+	}
+
+	if r, err := nc.Exec(); err != nil {
+		t.Fatal(err)
+	} else if string(r) != "test" {
+		t.Fatalf("expected exec result test got %s", string(r))
+	}
+}

agent/input/discord/README.md

@@ -0,0 +1,22 @@
+# Discord input for micro-bot
+[Discord](https://discordapp.com) support for micro bot based on [discordgo](github.com/bwmarrin/discordgo).
+
+This was originally written by Aleksandr Tihomirov (@zet4) and can be found at https://github.com/zet4/micro-misc/.
+
+## Options
+### discord_token
+
+You have to supply an application token via `--discord_token`.
+
+Head over to Discord's [developer introduction](https://discordapp.com/developers/docs/intro)
+to learn how to create applications and how the API works.
+
+### discord_prefix
+
+Set a command prefix with `--discord_prefix`. The default prefix is `Micro `.
+You can mention the bot or use the prefix to run a command.
+
+### discord_whitelist
+
+Pass a list of comma-separated user IDs with `--discord_whitelist`. Only allow
+these users to use the bot.

agent/input/discord/conn.go

@@ -0,0 +1,94 @@
+package discord
+
+import (
+	"errors"
+	"strings"
+	"sync"
+
+	"github.com/bwmarrin/discordgo"
+	"github.com/micro/go-micro/agent/input"
+	"github.com/micro/go-micro/util/log"
+)
+
+type discordConn struct {
+	master *discordInput
+	exit   chan struct{}
+	recv   chan *discordgo.Message
+
+	sync.Mutex
+}
+
+func newConn(master *discordInput) *discordConn {
+	conn := &discordConn{
+		master: master,
+		exit:   make(chan struct{}),
+		recv:   make(chan *discordgo.Message),
+	}
+
+	conn.master.session.AddHandler(func(s *discordgo.Session, m *discordgo.MessageCreate) {
+		if m.Author.ID == master.botID {
+			return
+		}
+
+		whitelisted := false
+		for _, ID := range conn.master.whitelist {
+			if m.Author.ID == ID {
+				whitelisted = true
+				break
+			}
+		}
+
+		if len(master.whitelist) > 0 && !whitelisted {
+			return
+		}
+
+		var valid bool
+		m.Message.Content, valid = conn.master.prefixfn(m.Message.Content)
+		if !valid {
+			return
+		}
+
+		conn.recv <- m.Message
+	})
+
+	return conn
+}
+
+func (dc *discordConn) Recv(event *input.Event) error {
+	for {
+		select {
+		case <-dc.exit:
+			return errors.New("connection closed")
+		case msg := <-dc.recv:
+
+			event.From = msg.ChannelID + ":" + msg.Author.ID
+			event.To = dc.master.botID
+			event.Type = input.TextEvent
+			event.Data = []byte(msg.Content)
+			return nil
+		}
+	}
+}
+
+func (dc *discordConn) Send(e *input.Event) error {
+	fields := strings.Split(e.To, ":")
+	_, err := dc.master.session.ChannelMessageSend(fields[0], string(e.Data))
+	if err != nil {
+		log.Log("[bot][loop][send]", err)
+	}
+	return nil
+}
+
+func (dc *discordConn) Close() error {
+	if err := dc.master.session.Close(); err != nil {
+		return err
+	}
+
+	select {
+	case <-dc.exit:
+		return nil
+	default:
+		close(dc.exit)
+	}
+	return nil
+}

agent/input/discord/discord.go

@@ -0,0 +1,153 @@
+package discord
+
+import (
+	"fmt"
+	"sync"
+
+	"errors"
+	"strings"
+
+	"github.com/bwmarrin/discordgo"
+	"github.com/micro/cli"
+	"github.com/micro/go-micro/agent/input"
+)
+
+func init() {
+	input.Inputs["discord"] = newInput()
+}
+
+func newInput() *discordInput {
+	return &discordInput{}
+}
+
+type discordInput struct {
+	token     string
+	whitelist []string
+	prefix    string
+	prefixfn  func(string) (string, bool)
+	botID     string
+
+	session *discordgo.Session
+
+	sync.Mutex
+	running bool
+	exit    chan struct{}
+}
+
+func (d *discordInput) Flags() []cli.Flag {
+	return []cli.Flag{
+		cli.StringFlag{
+			Name:   "discord_token",
+			EnvVar: "MICRO_DISCORD_TOKEN",
+			Usage:  "Discord token (prefix with Bot if it's for bot account)",
+		},
+		cli.StringFlag{
+			Name:   "discord_whitelist",
+			EnvVar: "MICRO_DISCORD_WHITELIST",
+			Usage:  "Discord Whitelist (seperated by ,)",
+		},
+		cli.StringFlag{
+			Name:   "discord_prefix",
+			Usage:  "Discord Prefix",
+			EnvVar: "MICRO_DISCORD_PREFIX",
+			Value:  "Micro ",
+		},
+	}
+}
+
+func (d *discordInput) Init(ctx *cli.Context) error {
+	token := ctx.String("discord_token")
+	whitelist := ctx.String("discord_whitelist")
+	prefix := ctx.String("discord_prefix")
+
+	if len(token) == 0 {
+		return errors.New("require token")
+	}
+
+	d.token = token
+	d.prefix = prefix
+
+	if len(whitelist) > 0 {
+		d.whitelist = strings.Split(whitelist, ",")
+	}
+
+	return nil
+}
+
+func (d *discordInput) Start() error {
+	if len(d.token) == 0 {
+		return errors.New("missing discord configuration")
+	}
+
+	d.Lock()
+	defer d.Unlock()
+
+	if d.running {
+		return nil
+	}
+
+	var err error
+	d.session, err = discordgo.New(d.token)
+	if err != nil {
+		return err
+	}
+
+	u, err := d.session.User("@me")
+	if err != nil {
+		return err
+	}
+
+	d.botID = u.ID
+	d.prefixfn = CheckPrefixFactory(fmt.Sprintf("<@%s> ", d.botID), fmt.Sprintf("<@!%s> ", d.botID), d.prefix)
+
+	d.exit = make(chan struct{})
+	d.running = true
+
+	return nil
+}
+
+func (d *discordInput) Stream() (input.Conn, error) {
+	d.Lock()
+	defer d.Unlock()
+	if !d.running {
+		return nil, errors.New("not running")
+	}
+
+	//Fire-n-forget close just in case...
+	d.session.Close()
+
+	conn := newConn(d)
+	if err := d.session.Open(); err != nil {
+		return nil, err
+	}
+	return conn, nil
+}
+
+func (d *discordInput) Stop() error {
+	d.Lock()
+	defer d.Unlock()
+
+	if !d.running {
+		return nil
+	}
+
+	close(d.exit)
+	d.running = false
+	return nil
+}
+
+func (d *discordInput) String() string {
+	return "discord"
+}
+
+// CheckPrefixFactory Creates a prefix checking function and stuff.
+func CheckPrefixFactory(prefixes ...string) func(string) (string, bool) {
+	return func(content string) (string, bool) {
+		for _, prefix := range prefixes {
+			if strings.HasPrefix(content, prefix) {
+				return strings.TrimPrefix(content, prefix), true
+			}
+		}
+		return "", false
+	}
+}

agent/input/input.go

@@ -0,0 +1,55 @@
+// Package input is an interface for bot inputs
+package input
+
+import (
+	"github.com/micro/cli"
+)
+
+type EventType string
+
+const (
+	TextEvent EventType = "text"
+)
+
+var (
+	// Inputs keyed by name
+	// Example slack or hipchat
+	Inputs = map[string]Input{}
+)
+
+// Event is the unit sent and received
+type Event struct {
+	Type EventType
+	From string
+	To   string
+	Data []byte
+	Meta map[string]interface{}
+}
+
+// Input is an interface for sources which
+// provide a way to communicate with the bot.
+// Slack, HipChat, XMPP, etc.
+type Input interface {
+	// Provide cli flags
+	Flags() []cli.Flag
+	// Initialise input using cli context
+	Init(*cli.Context) error
+	// Stream events from the input
+	Stream() (Conn, error)
+	// Start the input
+	Start() error
+	// Stop the input
+	Stop() error
+	// name of the input
+	String() string
+}
+
+// Conn interface provides a way to
+// send and receive events. Send and
+// Recv both block until succeeding
+// or failing.
+type Conn interface {
+	Close() error
+	Recv(*Event) error
+	Send(*Event) error
+}

agent/input/slack/conn.go

@@ -0,0 +1,160 @@
+package slack
+
+import (
+	"errors"
+	"fmt"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/micro/go-micro/agent/input"
+	"github.com/nlopes/slack"
+)
+
+// Satisfies the input.Conn interface
+type slackConn struct {
+	auth *slack.AuthTestResponse
+	rtm  *slack.RTM
+	exit chan bool
+
+	sync.Mutex
+	names map[string]string
+}
+
+func (s *slackConn) run() {
+	// func retrieves user names and maps to IDs
+	setNames := func() {
+		names := make(map[string]string)
+		users, err := s.rtm.Client.GetUsers()
+		if err != nil {
+			return
+		}
+
+		for _, user := range users {
+			names[user.ID] = user.Name
+		}
+
+		s.Lock()
+		s.names = names
+		s.Unlock()
+	}
+
+	setNames()
+
+	t := time.NewTicker(time.Minute)
+	defer t.Stop()
+
+	for {
+		select {
+		case <-s.exit:
+			return
+		case <-t.C:
+			setNames()
+		}
+	}
+}
+
+func (s *slackConn) getName(id string) string {
+	s.Lock()
+	name := s.names[id]
+	s.Unlock()
+	return name
+}
+
+func (s *slackConn) Close() error {
+	select {
+	case <-s.exit:
+		return nil
+	default:
+		close(s.exit)
+	}
+	return nil
+}
+
+func (s *slackConn) Recv(event *input.Event) error {
+	if event == nil {
+		return errors.New("event cannot be nil")
+	}
+
+	for {
+		select {
+		case <-s.exit:
+			return errors.New("connection closed")
+		case e := <-s.rtm.IncomingEvents:
+			switch ev := e.Data.(type) {
+			case *slack.MessageEvent:
+				// only accept type message
+				if ev.Type != "message" {
+					continue
+				}
+
+				// only accept DMs or messages to me
+				switch {
+				case strings.HasPrefix(ev.Channel, "D"):
+				case strings.HasPrefix(ev.Text, s.auth.User):
+				case strings.HasPrefix(ev.Text, fmt.Sprintf("<@%s>", s.auth.UserID)):
+				default:
+					continue
+				}
+
+				// Strip username from text
+				switch {
+				case strings.HasPrefix(ev.Text, s.auth.User):
+					args := strings.Split(ev.Text, " ")[1:]
+					ev.Text = strings.Join(args, " ")
+					event.To = s.auth.User
+				case strings.HasPrefix(ev.Text, fmt.Sprintf("<@%s>", s.auth.UserID)):
+					args := strings.Split(ev.Text, " ")[1:]
+					ev.Text = strings.Join(args, " ")
+					event.To = s.auth.UserID
+				}
+
+				if event.Meta == nil {
+					event.Meta = make(map[string]interface{})
+				}
+
+				// fill in the blanks
+				event.From = ev.Channel + ":" + ev.User
+				event.Type = input.TextEvent
+				event.Data = []byte(ev.Text)
+				event.Meta["reply"] = ev
+				return nil
+			case *slack.InvalidAuthEvent:
+				return errors.New("invalid credentials")
+			}
+		}
+	}
+}
+
+func (s *slackConn) Send(event *input.Event) error {
+	var channel, message, name string
+
+	if len(event.To) == 0 {
+		return errors.New("require Event.To")
+	}
+
+	parts := strings.Split(event.To, ":")
+
+	if len(parts) == 2 {
+		channel = parts[0]
+		name = s.getName(parts[1])
+		// try using reply meta
+	} else if ev, ok := event.Meta["reply"]; ok {
+		channel = ev.(*slack.MessageEvent).Channel
+		name = s.getName(ev.(*slack.MessageEvent).User)
+	}
+
+	// don't know where to send the message
+	if len(channel) == 0 {
+		return errors.New("could not determine who message is to")
+	}
+
+	if len(name) == 0 || strings.HasPrefix(channel, "D") {
+		message = string(event.Data)
+	} else {
+		message = fmt.Sprintf("@%s: %s", name, string(event.Data))
+	}
+
+	s.rtm.SendMessage(s.rtm.NewOutgoingMessage(message, channel))
+	return nil
+}

agent/input/slack/slack.go

@@ -0,0 +1,147 @@
+package slack
+
+import (
+	"errors"
+	"sync"
+
+	"github.com/micro/cli"
+	"github.com/micro/go-micro/agent/input"
+	"github.com/nlopes/slack"
+)
+
+type slackInput struct {
+	debug bool
+	token string
+
+	sync.Mutex
+	running bool
+	exit    chan bool
+
+	api *slack.Client
+}
+
+func init() {
+	input.Inputs["slack"] = NewInput()
+}
+
+func (p *slackInput) Flags() []cli.Flag {
+	return []cli.Flag{
+		cli.BoolFlag{
+			Name:   "slack_debug",
+			Usage:  "Slack debug output",
+			EnvVar: "MICRO_SLACK_DEBUG",
+		},
+		cli.StringFlag{
+			Name:   "slack_token",
+			Usage:  "Slack token",
+			EnvVar: "MICRO_SLACK_TOKEN",
+		},
+	}
+}
+
+func (p *slackInput) Init(ctx *cli.Context) error {
+	debug := ctx.Bool("slack_debug")
+	token := ctx.String("slack_token")
+
+	if len(token) == 0 {
+		return errors.New("missing slack token")
+	}
+
+	p.debug = debug
+	p.token = token
+
+	return nil
+}
+
+func (p *slackInput) Stream() (input.Conn, error) {
+	p.Lock()
+	defer p.Unlock()
+
+	if !p.running {
+		return nil, errors.New("not running")
+	}
+
+	// test auth
+	auth, err := p.api.AuthTest()
+	if err != nil {
+		return nil, err
+	}
+
+	rtm := p.api.NewRTM()
+	exit := make(chan bool)
+
+	go rtm.ManageConnection()
+
+	go func() {
+		select {
+		case <-p.exit:
+			select {
+			case <-exit:
+				return
+			default:
+				close(exit)
+			}
+		case <-exit:
+		}
+
+		rtm.Disconnect()
+	}()
+
+	conn := &slackConn{
+		auth:  auth,
+		rtm:   rtm,
+		exit:  exit,
+		names: make(map[string]string),
+	}
+
+	go conn.run()
+
+	return conn, nil
+}
+
+func (p *slackInput) Start() error {
+	if len(p.token) == 0 {
+		return errors.New("missing slack token")
+	}
+
+	p.Lock()
+	defer p.Unlock()
+
+	if p.running {
+		return nil
+	}
+
+	api := slack.New(p.token, slack.OptionDebug(p.debug))
+
+	// test auth
+	_, err := api.AuthTest()
+	if err != nil {
+		return err
+	}
+
+	p.api = api
+	p.exit = make(chan bool)
+	p.running = true
+	return nil
+}
+
+func (p *slackInput) Stop() error {
+	p.Lock()
+	defer p.Unlock()
+
+	if !p.running {
+		return nil
+	}
+
+	close(p.exit)
+	p.running = false
+	return nil
+}
+
+func (p *slackInput) String() string {
+	return "slack"
+}
+
+func NewInput() input.Input {
+	return &slackInput{}
+}

agent/input/telegram/README.md

@@ -0,0 +1,18 @@
+# Telegram Messenger input for micro bot
+[Telegram](https://telegram.org) support for micro bot based on [telegram-bot-api](https://github.com/go-telegram-bot-api/telegram-bot-api).
+
+## Options
+### --telegram_token (required)
+
+Sets bot's token for interacting with API.
+
+Head over to Telegram's [API documentation](https://core.telegram.org/bots/api)
+to learn how to create bots and how the API works.
+
+### --telegram_debug
+
+Sets the debug flag to make the bot's output verbose.
+
+### --telegram_whitelist
+
+Sets a list of comma-separated nicknames (without @ symbol in the beginning) for interacting with bot. Only these users can use the bot.

agent/input/telegram/conn.go

@@ -0,0 +1,115 @@
+package telegram
+
+import (
+	"errors"
+	"strings"
+	"sync"
+
+	"github.com/forestgiant/sliceutil"
+	"github.com/micro/go-micro/agent/input"
+	"github.com/micro/go-micro/util/log"
+	"gopkg.in/telegram-bot-api.v4"
+)
+
+type telegramConn struct {
+	input *telegramInput
+
+	recv <-chan tgbotapi.Update
+	exit chan bool
+
+	syncCond *sync.Cond
+	mutex    sync.Mutex
+}
+
+func newConn(input *telegramInput) (*telegramConn, error) {
+	conn := &telegramConn{
+		input: input,
+	}
+
+	conn.syncCond = sync.NewCond(&conn.mutex)
+
+	go conn.run()
+
+	return conn, nil
+}
+
+func (tc *telegramConn) run() {
+	u := tgbotapi.NewUpdate(0)
+	u.Timeout = 60
+	updates, err := tc.input.api.GetUpdatesChan(u)
+	if err != nil {
+		return
+	}
+
+	tc.recv = updates
+	tc.syncCond.Signal()
+
+	for {
+		select {
+		case <-tc.exit:
+			return
+		}
+	}
+}
+
+func (tc *telegramConn) Close() error {
+	return nil
+}
+
+func (tc *telegramConn) Recv(event *input.Event) error {
+	if event == nil {
+		return errors.New("event cannot be nil")
+	}
+
+	for {
+		if tc.recv == nil {
+			tc.mutex.Lock()
+			tc.syncCond.Wait()
+		}
+
+		update := <-tc.recv
+
+		if update.Message == nil || (len(tc.input.whitelist) > 0 && !sliceutil.Contains(tc.input.whitelist, update.Message.From.UserName)) {
+			continue
+		}
+
+		if event.Meta == nil {
+			event.Meta = make(map[string]interface{})
+		}
+
+		event.Type = input.TextEvent
+		event.From = update.Message.From.UserName
+		event.To = tc.input.api.Self.UserName
+		event.Data = []byte(update.Message.Text)
+		event.Meta["chatId"] = update.Message.Chat.ID
+		event.Meta["chatType"] = update.Message.Chat.Type
+		event.Meta["messageId"] = update.Message.MessageID
+
+		return nil
+	}
+}
+
+func (tc *telegramConn) Send(event *input.Event) error {
+	messageText := strings.TrimSpace(string(event.Data))
+
+	chatId := event.Meta["chatId"].(int64)
+	chatType := ChatType(event.Meta["chatType"].(string))
+
+	msgConfig := tgbotapi.NewMessage(chatId, messageText)
+	msgConfig.ParseMode = tgbotapi.ModeHTML
+
+	if sliceutil.Contains([]ChatType{Group, Supergroup}, chatType) {
+		msgConfig.ReplyToMessageID = event.Meta["messageId"].(int)
+	}
+
+	_, err := tc.input.api.Send(msgConfig)
+
+	if err != nil {
+		// probably it could be because of nested HTML tags -- telegram doesn't allow nested tags
+		log.Log("[telegram][Send] error:", err)
+		msgConfig.Text = "This bot couldn't send the response (Internal error)"
+		tc.input.api.Send(msgConfig)
+	}
+
+	return nil
+}

agent/input/telegram/telegram.go

@@ -0,0 +1,101 @@
+package telegram
+
+import (
+	"errors"
+	"strings"
+	"sync"
+
+	"github.com/micro/cli"
+	"github.com/micro/go-micro/agent/input"
+	"gopkg.in/telegram-bot-api.v4"
+)
+
+type telegramInput struct {
+	sync.Mutex
+
+	debug     bool
+	token     string
+	whitelist []string
+
+	api *tgbotapi.BotAPI
+}
+
+type ChatType string
+
+const (
+	Private    ChatType = "private"
+	Group      ChatType = "group"
+	Supergroup ChatType = "supergroup"
+)
+
+func init() {
+	input.Inputs["telegram"] = &telegramInput{}
+}
+
+func (ti *telegramInput) Flags() []cli.Flag {
+	return []cli.Flag{
+		cli.BoolFlag{
+			Name:   "telegram_debug",
+			EnvVar: "MICRO_TELEGRAM_DEBUG",
+			Usage:  "Telegram debug output",
+		},
+		cli.StringFlag{
+			Name:   "telegram_token",
+			EnvVar: "MICRO_TELEGRAM_TOKEN",
+			Usage:  "Telegram token",
+		},
+		cli.StringFlag{
+			Name:   "telegram_whitelist",
+			EnvVar: "MICRO_TELEGRAM_WHITELIST",
+			Usage:  "Telegram bot's users (comma-separated values)",
+		},
+	}
+}
+
+func (ti *telegramInput) Init(ctx *cli.Context) error {
+	ti.debug = ctx.Bool("telegram_debug")
+	ti.token = ctx.String("telegram_token")
+
+	whitelist := ctx.String("telegram_whitelist")
+
+	if whitelist != "" {
+		ti.whitelist = strings.Split(whitelist, ",")
+	}
+
+	if len(ti.token) == 0 {
+		return errors.New("missing telegram token")
+	}
+
+	return nil
+}
+
+func (ti *telegramInput) Stream() (input.Conn, error) {
+	ti.Lock()
+	defer ti.Unlock()
+
+	return newConn(ti)
+}
+
+func (ti *telegramInput) Start() error {
+	ti.Lock()
+	defer ti.Unlock()
+
+	api, err := tgbotapi.NewBotAPI(ti.token)
+	if err != nil {
+		return err
+	}
+
+	ti.api = api
+
+	api.Debug = ti.debug
+
+	return nil
+}
+
+func (ti *telegramInput) Stop() error {
+	return nil
+}
+
+func (p *telegramInput) String() string {
+	return "telegram"
+}

agent/proto/bot.micro.go

@@ -0,0 +1,118 @@
+// Code generated by protoc-gen-micro. DO NOT EDIT.
+// source: github.com/micro/go-bot/proto/bot.proto
+
+/*
+Package go_micro_bot is a generated protocol buffer package.
+
+It is generated from these files:
+	github.com/micro/go-bot/proto/bot.proto
+
+It has these top-level messages:
+	HelpRequest
+	HelpResponse
+	ExecRequest
+	ExecResponse
+*/
+package go_micro_bot
+
+import proto "github.com/golang/protobuf/proto"
+import fmt "fmt"
+import math "math"
+
+import (
+	context "context"
+	client "github.com/micro/go-micro/client"
+	server "github.com/micro/go-micro/server"
+)
+
+// Reference imports to suppress errors if they are not otherwise used.
+var _ = proto.Marshal
+var _ = fmt.Errorf
+var _ = math.Inf
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the proto package it is being compiled against.
+// A compilation error at this line likely means your copy of the
+// proto package needs to be updated.
+const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package
+
+// Reference imports to suppress errors if they are not otherwise used.
+var _ context.Context
+var _ client.Option
+var _ server.Option
+
+// Client API for Command service
+
+type CommandService interface {
+	Help(ctx context.Context, in *HelpRequest, opts ...client.CallOption) (*HelpResponse, error)
+	Exec(ctx context.Context, in *ExecRequest, opts ...client.CallOption) (*ExecResponse, error)
+}
+
+type commandService struct {
+	c    client.Client
+	name string
+}
+
+func NewCommandService(name string, c client.Client) CommandService {
+	if c == nil {
+		c = client.NewClient()
+	}
+	if len(name) == 0 {
+		name = "go.micro.bot"
+	}
+	return &commandService{
+		c:    c,
+		name: name,
+	}
+}
+
+func (c *commandService) Help(ctx context.Context, in *HelpRequest, opts ...client.CallOption) (*HelpResponse, error) {
+	req := c.c.NewRequest(c.name, "Command.Help", in)
+	out := new(HelpResponse)
+	err := c.c.Call(ctx, req, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *commandService) Exec(ctx context.Context, in *ExecRequest, opts ...client.CallOption) (*ExecResponse, error) {
+	req := c.c.NewRequest(c.name, "Command.Exec", in)
+	out := new(ExecResponse)
+	err := c.c.Call(ctx, req, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+// Server API for Command service
+
+type CommandHandler interface {
+	Help(context.Context, *HelpRequest, *HelpResponse) error
+	Exec(context.Context, *ExecRequest, *ExecResponse) error
+}
+
+func RegisterCommandHandler(s server.Server, hdlr CommandHandler, opts ...server.HandlerOption) error {
+	type _command interface {
+		Help(ctx context.Context, in *HelpRequest, out *HelpResponse) error
+		Exec(ctx context.Context, in *ExecRequest, out *ExecResponse) error
+	}
+	type Command struct {
+		_command
+	}
+	h := &commandHandler{hdlr}
+	return s.Handle(s.NewHandler(&Command{h}, opts...))
+}
+
+type commandHandler struct {
+	CommandHandler
+}
+
+func (h *commandHandler) Help(ctx context.Context, in *HelpRequest, out *HelpResponse) error {
+	return h.CommandHandler.Help(ctx, in, out)
+}
+
+func (h *commandHandler) Exec(ctx context.Context, in *ExecRequest, out *ExecResponse) error {
+	return h.CommandHandler.Exec(ctx, in, out)
+}

agent/proto/bot.pb.go

@@ -0,0 +1,210 @@
+// Code generated by protoc-gen-go. DO NOT EDIT.
+// source: github.com/micro/go-bot/proto/bot.proto
+
+package go_micro_bot
+
+import proto "github.com/golang/protobuf/proto"
+import fmt "fmt"
+import math "math"
+
+// Reference imports to suppress errors if they are not otherwise used.
+var _ = proto.Marshal
+var _ = fmt.Errorf
+var _ = math.Inf
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the proto package it is being compiled against.
+// A compilation error at this line likely means your copy of the
+// proto package needs to be updated.
+const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package
+
+type HelpRequest struct {
+	XXX_NoUnkeyedLiteral struct{} `json:"-"`
+	XXX_unrecognized     []byte   `json:"-"`
+	XXX_sizecache        int32    `json:"-"`
+}
+
+func (m *HelpRequest) Reset()         { *m = HelpRequest{} }
+func (m *HelpRequest) String() string { return proto.CompactTextString(m) }
+func (*HelpRequest) ProtoMessage()    {}
+func (*HelpRequest) Descriptor() ([]byte, []int) {
+	return fileDescriptor_bot_654832eab83ed4b5, []int{0}
+}
+func (m *HelpRequest) XXX_Unmarshal(b []byte) error {
+	return xxx_messageInfo_HelpRequest.Unmarshal(m, b)
+}
+func (m *HelpRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	return xxx_messageInfo_HelpRequest.Marshal(b, m, deterministic)
+}
+func (dst *HelpRequest) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_HelpRequest.Merge(dst, src)
+}
+func (m *HelpRequest) XXX_Size() int {
+	return xxx_messageInfo_HelpRequest.Size(m)
+}
+func (m *HelpRequest) XXX_DiscardUnknown() {
+	xxx_messageInfo_HelpRequest.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_HelpRequest proto.InternalMessageInfo
+
+type HelpResponse struct {
+	Usage                string   `protobuf:"bytes,1,opt,name=usage,proto3" json:"usage,omitempty"`
+	Description          string   `protobuf:"bytes,2,opt,name=description,proto3" json:"description,omitempty"`
+	XXX_NoUnkeyedLiteral struct{} `json:"-"`
+	XXX_unrecognized     []byte   `json:"-"`
+	XXX_sizecache        int32    `json:"-"`
+}
+
+func (m *HelpResponse) Reset()         { *m = HelpResponse{} }
+func (m *HelpResponse) String() string { return proto.CompactTextString(m) }
+func (*HelpResponse) ProtoMessage()    {}
+func (*HelpResponse) Descriptor() ([]byte, []int) {
+	return fileDescriptor_bot_654832eab83ed4b5, []int{1}
+}
+func (m *HelpResponse) XXX_Unmarshal(b []byte) error {
+	return xxx_messageInfo_HelpResponse.Unmarshal(m, b)
+}
+func (m *HelpResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	return xxx_messageInfo_HelpResponse.Marshal(b, m, deterministic)
+}
+func (dst *HelpResponse) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_HelpResponse.Merge(dst, src)
+}
+func (m *HelpResponse) XXX_Size() int {
+	return xxx_messageInfo_HelpResponse.Size(m)
+}
+func (m *HelpResponse) XXX_DiscardUnknown() {
+	xxx_messageInfo_HelpResponse.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_HelpResponse proto.InternalMessageInfo
+
+func (m *HelpResponse) GetUsage() string {
+	if m != nil {
+		return m.Usage
+	}
+	return ""
+}
+
+func (m *HelpResponse) GetDescription() string {
+	if m != nil {
+		return m.Description
+	}
+	return ""
+}
+
+type ExecRequest struct {
+	Args                 []string `protobuf:"bytes,1,rep,name=args,proto3" json:"args,omitempty"`
+	XXX_NoUnkeyedLiteral struct{} `json:"-"`
+	XXX_unrecognized     []byte   `json:"-"`
+	XXX_sizecache        int32    `json:"-"`
+}
+
+func (m *ExecRequest) Reset()         { *m = ExecRequest{} }
+func (m *ExecRequest) String() string { return proto.CompactTextString(m) }
+func (*ExecRequest) ProtoMessage()    {}
+func (*ExecRequest) Descriptor() ([]byte, []int) {
+	return fileDescriptor_bot_654832eab83ed4b5, []int{2}
+}
+func (m *ExecRequest) XXX_Unmarshal(b []byte) error {
+	return xxx_messageInfo_ExecRequest.Unmarshal(m, b)
+}
+func (m *ExecRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	return xxx_messageInfo_ExecRequest.Marshal(b, m, deterministic)
+}
+func (dst *ExecRequest) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_ExecRequest.Merge(dst, src)
+}
+func (m *ExecRequest) XXX_Size() int {
+	return xxx_messageInfo_ExecRequest.Size(m)
+}
+func (m *ExecRequest) XXX_DiscardUnknown() {
+	xxx_messageInfo_ExecRequest.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_ExecRequest proto.InternalMessageInfo
+
+func (m *ExecRequest) GetArgs() []string {
+	if m != nil {
+		return m.Args
+	}
+	return nil
+}
+
+type ExecResponse struct {
+	Result               []byte   `protobuf:"bytes,1,opt,name=result,proto3" json:"result,omitempty"`
+	Error                string   `protobuf:"bytes,2,opt,name=error,proto3" json:"error,omitempty"`
+	XXX_NoUnkeyedLiteral struct{} `json:"-"`
+	XXX_unrecognized     []byte   `json:"-"`
+	XXX_sizecache        int32    `json:"-"`
+}
+
+func (m *ExecResponse) Reset()         { *m = ExecResponse{} }
+func (m *ExecResponse) String() string { return proto.CompactTextString(m) }
+func (*ExecResponse) ProtoMessage()    {}
+func (*ExecResponse) Descriptor() ([]byte, []int) {
+	return fileDescriptor_bot_654832eab83ed4b5, []int{3}
+}
+func (m *ExecResponse) XXX_Unmarshal(b []byte) error {
+	return xxx_messageInfo_ExecResponse.Unmarshal(m, b)
+}
+func (m *ExecResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	return xxx_messageInfo_ExecResponse.Marshal(b, m, deterministic)
+}
+func (dst *ExecResponse) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_ExecResponse.Merge(dst, src)
+}
+func (m *ExecResponse) XXX_Size() int {
+	return xxx_messageInfo_ExecResponse.Size(m)
+}
+func (m *ExecResponse) XXX_DiscardUnknown() {
+	xxx_messageInfo_ExecResponse.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_ExecResponse proto.InternalMessageInfo
+
+func (m *ExecResponse) GetResult() []byte {
+	if m != nil {
+		return m.Result
+	}
+	return nil
+}
+
+func (m *ExecResponse) GetError() string {
+	if m != nil {
+		return m.Error
+	}
+	return ""
+}
+
+func init() {
+	proto.RegisterType((*HelpRequest)(nil), "go.micro.bot.HelpRequest")
+	proto.RegisterType((*HelpResponse)(nil), "go.micro.bot.HelpResponse")
+	proto.RegisterType((*ExecRequest)(nil), "go.micro.bot.ExecRequest")
+	proto.RegisterType((*ExecResponse)(nil), "go.micro.bot.ExecResponse")
+}
+
+func init() {
+	proto.RegisterFile("github.com/micro/go-bot/proto/bot.proto", fileDescriptor_bot_654832eab83ed4b5)
+}
+
+var fileDescriptor_bot_654832eab83ed4b5 = []byte{
+	// 246 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x6c, 0x90, 0x41, 0x4b, 0xc4, 0x30,
+	0x10, 0x85, 0xb7, 0xba, 0xae, 0xec, 0xb4, 0x5e, 0x82, 0x48, 0xdd, 0x53, 0xcd, 0xc5, 0xbd, 0x98,
+	0x82, 0x5e, 0x05, 0x0f, 0xa2, 0x78, 0xee, 0x3f, 0x68, 0xba, 0x43, 0x2c, 0x6c, 0x3b, 0x35, 0x99,
+	0x82, 0xff, 0xc1, 0x3f, 0x2d, 0x4d, 0x72, 0x08, 0xcb, 0xde, 0xe6, 0x65, 0x86, 0xf7, 0xbe, 0x17,
+	0x78, 0x34, 0x3d, 0x7f, 0xcf, 0x5a, 0x75, 0x34, 0xd4, 0x43, 0xdf, 0x59, 0xaa, 0x0d, 0x3d, 0x69,
+	0xe2, 0x7a, 0xb2, 0xc4, 0x54, 0x6b, 0x62, 0xe5, 0x27, 0x51, 0x18, 0x52, 0xfe, 0x40, 0x69, 0x62,
+	0x79, 0x03, 0xf9, 0x17, 0x1e, 0xa7, 0x06, 0x7f, 0x66, 0x74, 0x2c, 0x3f, 0xa1, 0x08, 0xd2, 0x4d,
+	0x34, 0x3a, 0x14, 0xb7, 0x70, 0x35, 0xbb, 0xd6, 0x60, 0x99, 0x55, 0xd9, 0x7e, 0xdb, 0x04, 0x21,
+	0x2a, 0xc8, 0x0f, 0xe8, 0x3a, 0xdb, 0x4f, 0xdc, 0xd3, 0x58, 0x5e, 0xf8, 0x5d, 0xfa, 0x24, 0x1f,
+	0x20, 0xff, 0xf8, 0xc5, 0x2e, 0xda, 0x0a, 0x01, 0xeb, 0xd6, 0x1a, 0x57, 0x66, 0xd5, 0xe5, 0x7e,
+	0xdb, 0xf8, 0x59, 0xbe, 0x42, 0x11, 0x4e, 0x62, 0xd4, 0x1d, 0x6c, 0x2c, 0xba, 0xf9, 0xc8, 0x3e,
+	0xab, 0x68, 0xa2, 0x5a, 0x10, 0xd0, 0x5a, 0xb2, 0x31, 0x26, 0x88, 0xe7, 0xbf, 0x0c, 0xae, 0xdf,
+	0x69, 0x18, 0xda, 0xf1, 0x20, 0xde, 0x60, 0xbd, 0x40, 0x8b, 0x7b, 0x95, 0x56, 0x53, 0x49, 0xaf,
+	0xdd, 0xee, 0xdc, 0x2a, 0x04, 0xcb, 0xd5, 0x62, 0xb0, 0xa0, 0x9c, 0x1a, 0x24, 0x0d, 0x4e, 0x0d,
+	0x52, 0x72, 0xb9, 0xd2, 0x1b, 0xff, 0xb5, 0x2f, 0xff, 0x01, 0x00, 0x00, 0xff, 0xff, 0xcb, 0x77,
+	0xdf, 0x28, 0x85, 0x01, 0x00, 0x00,
+}

agent/proto/bot.proto

@@ -0,0 +1,25 @@
+syntax = "proto3";
+
+package go.micro.bot;
+
+service Command {
+	rpc Help(HelpRequest) returns (HelpResponse) {};
+	rpc Exec(ExecRequest) returns (ExecResponse) {};
+}
+
+message HelpRequest {
+}
+
+message HelpResponse {
+	string usage = 1;
+	string description = 2;
+}
+
+message ExecRequest {
+	repeated string args = 1;
+}
+
+message ExecResponse {
+	bytes result = 1;
+	string error = 2;
+}

broker/http_broker.go

@@ -22,10 +22,10 @@ import (
 	"github.com/micro/go-micro/codec/json"
 	merr "github.com/micro/go-micro/errors"
 	"github.com/micro/go-micro/registry"
-	"github.com/micro/go-rcache"
-	maddr "github.com/micro/util/go/lib/addr"
-	mnet "github.com/micro/util/go/lib/net"
-	mls "github.com/micro/util/go/lib/tls"
+	"github.com/micro/go-micro/registry/cache"
+	maddr "github.com/micro/go-micro/util/addr"
+	mnet "github.com/micro/go-micro/util/net"
+	mls "github.com/micro/go-micro/util/tls"
 	"golang.org/x/net/http2"
 )
 
@@ -412,8 +412,8 @@ func (h *httpBroker) Connect() error {
 	if !ok {
 		reg = registry.DefaultRegistry
 	}
-	// set rcache
-	h.r = rcache.New(reg)
+	// set cache
+	h.r = cache.New(reg)
 
 	// set running
 	h.running = true
@@ -432,8 +432,8 @@ func (h *httpBroker) Disconnect() error {
 	h.Lock()
 	defer h.Unlock()
 
-	// stop rcache
-	rc, ok := h.r.(rcache.Cache)
+	// stop cache
+	rc, ok := h.r.(cache.Cache)
 	if ok {
 		rc.Stop()
 	}
@@ -477,13 +477,13 @@ func (h *httpBroker) Init(opts ...Option) error {
 		reg = registry.DefaultRegistry
 	}
 
-	// get rcache
-	if rc, ok := h.r.(rcache.Cache); ok {
+	// get cache
+	if rc, ok := h.r.(cache.Cache); ok {
 		rc.Stop()
 	}
 
 	// set registry
-	h.r = rcache.New(reg)
+	h.r = cache.New(reg)
 
 	// reconfigure tls config
 	if c := h.opts.TLSConfig; c != nil {

broker/http_broker_test.go

@@ -7,8 +7,8 @@ import (
 
 	glog "github.com/go-log/log"
 	"github.com/google/uuid"
-	"github.com/micro/go-log"
 	"github.com/micro/go-micro/registry/memory"
+	"github.com/micro/go-micro/util/log"
 )
 
 func newTestRegistry() *memory.Registry {

broker/nats/context.go

@@ -0,0 +1,37 @@
+package nats
+
+import (
+	"context"
+
+	"github.com/micro/go-micro/broker"
+)
+
+// setSubscribeOption returns a function to setup a context with given value
+func setSubscribeOption(k, v interface{}) broker.SubscribeOption {
+	return func(o *broker.SubscribeOptions) {
+		if o.Context == nil {
+			o.Context = context.Background()
+		}
+		o.Context = context.WithValue(o.Context, k, v)
+	}
+}
+
+// setBrokerOption returns a function to setup a context with given value
+func setBrokerOption(k, v interface{}) broker.Option {
+	return func(o *broker.Options) {
+		if o.Context == nil {
+			o.Context = context.Background()
+		}
+		o.Context = context.WithValue(o.Context, k, v)
+	}
+}
+
+// setPublishOption returns a function to setup a context with given value
+func setPublishOption(k, v interface{}) broker.PublishOption {
+	return func(o *broker.PublishOptions) {
+		if o.Context == nil {
+			o.Context = context.Background()
+		}
+		o.Context = context.WithValue(o.Context, k, v)
+	}
+}

broker/nats/nats.go

@@ -0,0 +1,245 @@
+// Package nats provides a NATS broker
+package nats
+
+import (
+	"context"
+	"errors"
+	"strings"
+	"sync"
+
+	"github.com/micro/go-micro/broker"
+	"github.com/micro/go-micro/codec/json"
+	nats "github.com/nats-io/nats.go"
+)
+
+type natsBroker struct {
+	sync.RWMutex
+	addrs []string
+	conn  *nats.Conn
+	opts  broker.Options
+	nopts nats.Options
+	drain bool
+}
+
+type subscriber struct {
+	s     *nats.Subscription
+	opts  broker.SubscribeOptions
+	drain bool
+}
+
+type publication struct {
+	t string
+	m *broker.Message
+}
+
+func (p *publication) Topic() string {
+	return p.t
+}
+
+func (p *publication) Message() *broker.Message {
+	return p.m
+}
+
+func (p *publication) Ack() error {
+	// nats does not support acking
+	return nil
+}
+
+func (s *subscriber) Options() broker.SubscribeOptions {
+	return s.opts
+}
+
+func (s *subscriber) Topic() string {
+	return s.s.Subject
+}
+
+func (s *subscriber) Unsubscribe() error {
+	if s.drain {
+		return s.s.Drain()
+	}
+	return s.s.Unsubscribe()
+}
+
+func (n *natsBroker) Address() string {
+	if n.conn != nil && n.conn.IsConnected() {
+		return n.conn.ConnectedUrl()
+	}
+	if len(n.addrs) > 0 {
+		return n.addrs[0]
+	}
+
+	return ""
+}
+
+func setAddrs(addrs []string) []string {
+	var cAddrs []string
+	for _, addr := range addrs {
+		if len(addr) == 0 {
+			continue
+		}
+		if !strings.HasPrefix(addr, "nats://") {
+			addr = "nats://" + addr
+		}
+		cAddrs = append(cAddrs, addr)
+	}
+	if len(cAddrs) == 0 {
+		cAddrs = []string{nats.DefaultURL}
+	}
+	return cAddrs
+}
+
+func (n *natsBroker) Connect() error {
+	n.Lock()
+	defer n.Unlock()
+
+	status := nats.CLOSED
+	if n.conn != nil {
+		status = n.conn.Status()
+	}
+
+	switch status {
+	case nats.CONNECTED, nats.RECONNECTING, nats.CONNECTING:
+		return nil
+	default: // DISCONNECTED or CLOSED or DRAINING
+		opts := n.nopts
+		opts.Servers = n.addrs
+		opts.Secure = n.opts.Secure
+		opts.TLSConfig = n.opts.TLSConfig
+
+		// secure might not be set
+		if n.opts.TLSConfig != nil {
+			opts.Secure = true
+		}
+
+		c, err := opts.Connect()
+		if err != nil {
+			return err
+		}
+		n.conn = c
+		return nil
+	}
+}
+
+func (n *natsBroker) Disconnect() error {
+	n.RLock()
+	if n.drain {
+		n.conn.Drain()
+	} else {
+		n.conn.Close()
+	}
+	n.RUnlock()
+	return nil
+}
+
+func (n *natsBroker) Init(opts ...broker.Option) error {
+	for _, o := range opts {
+		o(&n.opts)
+	}
+	n.addrs = setAddrs(n.opts.Addrs)
+	return nil
+}
+
+func (n *natsBroker) Options() broker.Options {
+	return n.opts
+}
+
+func (n *natsBroker) Publish(topic string, msg *broker.Message, opts ...broker.PublishOption) error {
+	b, err := n.opts.Codec.Marshal(msg)
+	if err != nil {
+		return err
+	}
+	n.RLock()
+	defer n.RUnlock()
+	return n.conn.Publish(topic, b)
+}
+
+func (n *natsBroker) Subscribe(topic string, handler broker.Handler, opts ...broker.SubscribeOption) (broker.Subscriber, error) {
+	if n.conn == nil {
+		return nil, errors.New("not connected")
+	}
+
+	opt := broker.SubscribeOptions{
+		AutoAck: true,
+		Context: context.Background(),
+	}
+
+	for _, o := range opts {
+		o(&opt)
+	}
+
+	var drain bool
+	if _, ok := opt.Context.Value(drainSubscriptionKey{}).(bool); ok {
+		drain = true
+	}
+
+	fn := func(msg *nats.Msg) {
+		var m broker.Message
+		if err := n.opts.Codec.Unmarshal(msg.Data, &m); err != nil {
+			return
+		}
+		handler(&publication{m: &m, t: msg.Subject})
+	}
+
+	var sub *nats.Subscription
+	var err error
+
+	n.RLock()
+	if len(opt.Queue) > 0 {
+		sub, err = n.conn.QueueSubscribe(topic, opt.Queue, fn)
+	} else {
+		sub, err = n.conn.Subscribe(topic, fn)
+	}
+	n.RUnlock()
+	if err != nil {
+		return nil, err
+	}
+	return &subscriber{s: sub, opts: opt, drain: drain}, nil
+}
+
+func (n *natsBroker) String() string {
+	return "nats"
+}
+
+func NewBroker(opts ...broker.Option) broker.Broker {
+	options := broker.Options{
+		// Default codec
+		Codec:   json.Marshaler{},
+		Context: context.Background(),
+	}
+
+	for _, o := range opts {
+		o(&options)
+	}
+
+	natsOpts := nats.GetDefaultOptions()
+	if n, ok := options.Context.Value(optionsKey{}).(nats.Options); ok {
+		natsOpts = n
+	}
+
+	var drain bool
+	if _, ok := options.Context.Value(drainSubscriptionKey{}).(bool); ok {
+		drain = true
+	}
+
+	// broker.Options have higher priority than nats.Options
+	// only if Addrs, Secure or TLSConfig were not set through a broker.Option
+	// we read them from nats.Option
+	if len(options.Addrs) == 0 {
+		options.Addrs = natsOpts.Servers
+	}
+
+	if !options.Secure {
+		options.Secure = natsOpts.Secure
+	}
+
+	if options.TLSConfig == nil {
+		options.TLSConfig = natsOpts.TLSConfig
+	}
+
+	return &natsBroker{
+		opts:  options,
+		nopts: natsOpts,
+		addrs: setAddrs(options.Addrs),
+		drain: drain,
+	}
+}

broker/nats/nats_test.go

@@ -0,0 +1,99 @@
+package nats
+
+import (
+	"fmt"
+	"testing"
+
+	"github.com/micro/go-micro/broker"
+	nats "github.com/nats-io/nats.go"
+)
+
+var addrTestCases = []struct {
+	name        string
+	description string
+	addrs       map[string]string // expected address : set address
+}{
+	{
+		"brokerOpts",
+		"set broker addresses through a broker.Option in constructor",
+		map[string]string{
+			"nats://192.168.10.1:5222": "192.168.10.1:5222",
+			"nats://10.20.10.0:4222":   "10.20.10.0:4222"},
+	},
+	{
+		"brokerInit",
+		"set broker addresses through a broker.Option in broker.Init()",
+		map[string]string{
+			"nats://192.168.10.1:5222": "192.168.10.1:5222",
+			"nats://10.20.10.0:4222":   "10.20.10.0:4222"},
+	},
+	{
+		"natsOpts",
+		"set broker addresses through the nats.Option in constructor",
+		map[string]string{
+			"nats://192.168.10.1:5222": "192.168.10.1:5222",
+			"nats://10.20.10.0:4222":   "10.20.10.0:4222"},
+	},
+	{
+		"default",
+		"check if default Address is set correctly",
+		map[string]string{
+			"nats://localhost:4222": "",
+		},
+	},
+}
+
+// TestInitAddrs tests issue #100. Ensures that if the addrs is set by an option in init it will be used.
+func TestInitAddrs(t *testing.T) {
+
+	for _, tc := range addrTestCases {
+		t.Run(fmt.Sprintf("%s: %s", tc.name, tc.description), func(t *testing.T) {
+
+			var br broker.Broker
+			var addrs []string
+
+			for _, addr := range tc.addrs {
+				addrs = append(addrs, addr)
+			}
+
+			switch tc.name {
+			case "brokerOpts":
+				// we know that there are just two addrs in the dict
+				br = NewBroker(broker.Addrs(addrs[0], addrs[1]))
+				br.Init()
+			case "brokerInit":
+				br = NewBroker()
+				// we know that there are just two addrs in the dict
+				br.Init(broker.Addrs(addrs[0], addrs[1]))
+			case "natsOpts":
+				nopts := nats.GetDefaultOptions()
+				nopts.Servers = addrs
+				br = NewBroker(Options(nopts))
+				br.Init()
+			case "default":
+				br = NewBroker()
+				br.Init()
+			}
+
+			natsBroker, ok := br.(*natsBroker)
+			if !ok {
+				t.Fatal("Expected broker to be of types *natsBroker")
+			}
+			// check if the same amount of addrs we set has actually been set, default
+			// have only 1 address nats://127.0.0.1:4222 (current nats code) or
+			// nats://localhost:4222 (older code version)
+			if len(natsBroker.addrs) != len(tc.addrs) && tc.name != "default" {
+				t.Errorf("Expected Addr count = %d, Actual Addr count = %d",
+					len(natsBroker.addrs), len(tc.addrs))
+			}
+
+			for _, addr := range natsBroker.addrs {
+				_, ok := tc.addrs[addr]
+				if !ok {
+					t.Errorf("Expected '%s' has not been set", addr)
+				}
+			}
+		})
+
+	}
+}

broker/nats/options.go

@@ -0,0 +1,25 @@
+package nats
+
+import (
+	"github.com/micro/go-micro/broker"
+	nats "github.com/nats-io/nats.go"
+)
+
+type optionsKey struct{}
+type drainConnectionKey struct{}
+type drainSubscriptionKey struct{}
+
+// Options accepts nats.Options
+func Options(opts nats.Options) broker.Option {
+	return setBrokerOption(optionsKey{}, opts)
+}
+
+// DrainConnection will drain subscription on close
+func DrainConnection() broker.Option {
+	return setBrokerOption(drainConnectionKey{}, true)
+}
+
+// DrainSubscription will drain pending messages when unsubscribe
+func DrainSubscription() broker.SubscribeOption {
+	return setSubscribeOption(drainSubscriptionKey{}, true)
+}

broker/options.go

@@ -44,10 +44,8 @@ type PublishOption func(*PublishOptions)
 
 type SubscribeOption func(*SubscribeOptions)
 
-type contextKeyT string
-
 var (
-	registryKey = contextKeyT("github.com/micro/go-micro/registry")
+	registryKey = "github.com/micro/go-micro/registry"
 )
 
 func NewSubscribeOptions(opts ...SubscribeOption) SubscribeOptions {

client/mock/mock.go

@@ -83,7 +83,11 @@ func (m *MockClient) Call(ctx context.Context, req client.Request, rsp interface
 		}
 		response := r.Response
 		if t := reflect.TypeOf(r.Response); t.Kind() == reflect.Func {
-			response = reflect.ValueOf(r.Response).Call([]reflect.Value{})[0].Interface()
+			var request []reflect.Value
+			if t.NumIn() == 1 {
+				request = append(request, reflect.ValueOf(req.Body()))
+			}
+			response = reflect.ValueOf(r.Response).Call(request)[0].Interface()
 		}
 
 		v.Set(reflect.ValueOf(response))

client/mock/mock_test.go

@@ -18,12 +18,18 @@ func TestClient(t *testing.T) {
 		{Endpoint: "Foo.Fail", Error: errors.InternalServerError("go.mock", "failed")},
 		{Endpoint: "Foo.Func", Response: func() string { return "string" }},
 		{Endpoint: "Foo.FuncStruct", Response: func() *TestResponse { return &TestResponse{Param: "aparam"} }},
+		{Endpoint: "Foo.FuncWithReqBody", Response: func(req interface{}) string {
+			if req.(map[string]string)["foo"] == "bar" {
+				return "string"
+			}
+			return "wrong"
+		}},
 	}
 
 	c := NewClient(Response("go.mock", response))
 
 	for _, r := range response {
-		req := c.NewRequest("go.mock", r.Endpoint, map[string]interface{}{"foo": "bar"})
+		req := c.NewRequest("go.mock", r.Endpoint, map[string]string{"foo": "bar"})
 		var rsp interface{}
 
 		err := c.Call(context.TODO(), req, &rsp)
@@ -33,6 +39,20 @@ func TestClient(t *testing.T) {
 		}
 
 		t.Log(rsp)
+		if r.Endpoint == "Foo.FuncWithReqBody" {
+			req := c.NewRequest("go.mock", r.Endpoint, map[string]string{"foo": "wrong"})
+			var rsp interface{}
+
+			err := c.Call(context.TODO(), req, &rsp)
+
+			if err != r.Error {
+				t.Fatalf("Expecter error %v got %v", r.Error, err)
+			}
+			if rsp.(string) != "wrong" {
+				t.Fatalf("Expecter response 'wrong' got %v", rsp)
+			}
+			t.Log(rsp)
+		}
 	}
 
 }

client/rpc_client.go

@@ -382,7 +382,7 @@ func (r *rpcClient) Call(ctx context.Context, request Request, response interfac
 		return err
 	}
 
-	ch := make(chan error, callOpts.Retries)
+	ch := make(chan error, callOpts.Retries+1)
 	var gerr error
 
 	for i := 0; i <= callOpts.Retries; i++ {
@@ -463,14 +463,14 @@ func (r *rpcClient) Stream(ctx context.Context, request Request, opts ...CallOpt
 		err    error
 	}
 
-	ch := make(chan response, callOpts.Retries)
+	ch := make(chan response, callOpts.Retries+1)
 	var grr error
 
 	for i := 0; i <= callOpts.Retries; i++ {
-		go func() {
+		go func(i int) {
 			s, err := call(i)
 			ch <- response{s, err}
-		}()
+		}(i)
 
 		select {
 		case <-ctx.Done():

cmd/cmd.go

@@ -10,14 +10,15 @@ import (
 	"time"
 
 	"github.com/micro/cli"
-	"github.com/micro/go-log"
 	"github.com/micro/go-micro/client"
 	"github.com/micro/go-micro/server"
+	"github.com/micro/go-micro/util/log"
 
 	// brokers
 	"github.com/micro/go-micro/broker"
 	"github.com/micro/go-micro/broker/http"
 	"github.com/micro/go-micro/broker/memory"
+	"github.com/micro/go-micro/broker/nats"
 
 	// registries
 	"github.com/micro/go-micro/registry"
@@ -71,6 +72,7 @@ var (
 		cli.IntFlag{
 			Name:   "client_retries",
 			EnvVar: "MICRO_CLIENT_RETRIES",
+			Value:  client.DefaultRetries,
 			Usage:  "Sets the client retries. Default: 1",
 		},
 		cli.IntFlag{
@@ -169,6 +171,7 @@ var (
 	DefaultBrokers = map[string]func(...broker.Option) broker.Broker{
 		"http":   http.NewBroker,
 		"memory": memory.NewBroker,
+		"nats":   nats.NewBroker,
 	}
 
 	DefaultClients = map[string]func(...client.Option) client.Client{

codec/proto/marshaler.go

@@ -14,6 +14,6 @@ func (Marshaler) Unmarshal(data []byte, v interface{}) error {
 	return proto.Unmarshal(data, v.(proto.Message))
 }
 
-func (Marshaler) Name() string {
+func (Marshaler) String() string {
 	return "proto"
 }

codec/text/text.go

@@ -0,0 +1,75 @@
+// Package text reads any text/* content-type
+package text
+
+import (
+	"fmt"
+	"io"
+	"io/ioutil"
+
+	"github.com/micro/go-micro/codec"
+)
+
+type Codec struct {
+	Conn io.ReadWriteCloser
+}
+
+// Frame gives us the ability to define raw data to send over the pipes
+type Frame struct {
+	Data []byte
+}
+
+func (c *Codec) ReadHeader(m *codec.Message, t codec.MessageType) error {
+	return nil
+}
+
+func (c *Codec) ReadBody(b interface{}) error {
+	// read bytes
+	buf, err := ioutil.ReadAll(c.Conn)
+	if err != nil {
+		return err
+	}
+
+	switch b.(type) {
+	case *[]byte:
+		v := b.(*[]byte)
+		*v = buf
+	case *Frame:
+		v := b.(*Frame)
+		v.Data = buf
+	default:
+		return fmt.Errorf("failed to read body: %v is not type of *[]byte", b)
+	}
+
+	return nil
+}
+
+func (c *Codec) Write(m *codec.Message, b interface{}) error {
+	var v []byte
+	switch b.(type) {
+	case *Frame:
+		v = b.(*Frame).Data
+	case *[]byte:
+		ve := b.(*[]byte)
+		v = *ve
+	case []byte:
+		v = b.([]byte)
+	default:
+		return fmt.Errorf("failed to write: %v is not type of *[]byte or []byte", b)
+	}
+	_, err := c.Conn.Write(v)
+	return err
+}
+
+func (c *Codec) Close() error {
+	return c.Conn.Close()
+}
+
+func (c *Codec) String() string {
+	return "bytes"
+}
+
+func NewCodec(c io.ReadWriteCloser) codec.Codec {
+	return &Codec{
+		Conn: c,
+	}
+}

config/README.md

@@ -0,0 +1,29 @@
+# Config [![GoDoc](https://godoc.org/github.com/micro/go-micro/config?status.svg)](https://godoc.org/github.com/micro/go-micro/config)
+
+Go Config is a pluggable dynamic config library.
+
+Most config in applications are statically configured or include complex logic to load from multiple sources. 
+Go Config makes this easy, pluggable and mergeable. You'll never have to deal with config in the same way again.
+
+## Features
+
+- **Dynamic Loading** - Load configuration from multiple source as and when needed. Go Config manages watching config sources 
+in the background and automatically merges and updates an in memory view. 
+
+- **Pluggable Sources** - Choose from any number of sources to load and merge config. The backend source is abstracted away into 
+a standard format consumed internally and decoded via encoders. Sources can be env vars, flags, file, etcd, k8s configmap, etc.
+
+- **Mergeable Config** - If you specify multiple sources of config, regardless of format, they will be merged and presented in 
+a single view. This massively simplifies priority order loading and changes based on environment.
+
+- **Observe Changes** - Optionally watch the config for changes to specific values. Hot reload your app using Go Config's watcher. 
+You don't have to handle ad-hoc hup reloading or whatever else, just keep reading the config and watch for changes if you need 
+to be notified.
+
+- **Sane Defaults** - In case config loads badly or is completely wiped away for some unknown reason, you can specify fallback 
+values when accessing any config values directly. This ensures you'll always be reading some sane default in the event of a problem.
+
+## Getting Started
+
+For detailed information or architecture, installation and general usage see the [docs](https://micro.mu/docs/go-config.html)
+

config/config.go

@@ -0,0 +1,94 @@
+// Package config is an interface for dynamic configuration.
+package config
+
+import (
+	"context"
+
+	"github.com/micro/go-micro/config/loader"
+	"github.com/micro/go-micro/config/reader"
+	"github.com/micro/go-micro/config/source"
+	"github.com/micro/go-micro/config/source/file"
+)
+
+// Config is an interface abstraction for dynamic configuration
+type Config interface {
+	// provide the reader.Values interface
+	reader.Values
+	// Stop the config loader/watcher
+	Close() error
+	// Load config sources
+	Load(source ...source.Source) error
+	// Force a source changeset sync
+	Sync() error
+	// Watch a value for changes
+	Watch(path ...string) (Watcher, error)
+}
+
+// Watcher is the config watcher
+type Watcher interface {
+	Next() (reader.Value, error)
+	Stop() error
+}
+
+type Options struct {
+	Loader loader.Loader
+	Reader reader.Reader
+	Source []source.Source
+
+	// for alternative data
+	Context context.Context
+}
+
+type Option func(o *Options)
+
+var (
+	// Default Config Manager
+	DefaultConfig = NewConfig()
+)
+
+// NewConfig returns new config
+func NewConfig(opts ...Option) Config {
+	return newConfig(opts...)
+}
+
+// Return config as raw json
+func Bytes() []byte {
+	return DefaultConfig.Bytes()
+}
+
+// Return config as a map
+func Map() map[string]interface{} {
+	return DefaultConfig.Map()
+}
+
+// Scan values to a go type
+func Scan(v interface{}) error {
+	return DefaultConfig.Scan(v)
+}
+
+// Force a source changeset sync
+func Sync() error {
+	return DefaultConfig.Sync()
+}
+
+// Get a value from the config
+func Get(path ...string) reader.Value {
+	return DefaultConfig.Get(path...)
+}
+
+// Load config sources
+func Load(source ...source.Source) error {
+	return DefaultConfig.Load(source...)
+}
+
+// Watch a value for changes
+func Watch(path ...string) (Watcher, error) {
+	return DefaultConfig.Watch(path...)
+}
+
+// LoadFile is short hand for creating a file source and loading it
+func LoadFile(path string) error {
+	return Load(file.NewSource(
+		file.WithPath(path),
+	))
+}

config/default.go

@@ -0,0 +1,253 @@
+package config
+
+import (
+	"bytes"
+	"sync"
+	"time"
+
+	"github.com/micro/go-micro/config/loader"
+	"github.com/micro/go-micro/config/loader/memory"
+	"github.com/micro/go-micro/config/reader"
+	"github.com/micro/go-micro/config/reader/json"
+	"github.com/micro/go-micro/config/source"
+)
+
+type config struct {
+	exit chan bool
+	opts Options
+
+	sync.RWMutex
+	// the current snapshot
+	snap *loader.Snapshot
+	// the current values
+	vals reader.Values
+}
+
+type watcher struct {
+	lw    loader.Watcher
+	rd    reader.Reader
+	path  []string
+	value reader.Value
+}
+
+func newConfig(opts ...Option) Config {
+	options := Options{
+		Loader: memory.NewLoader(),
+		Reader: json.NewReader(),
+	}
+
+	for _, o := range opts {
+		o(&options)
+	}
+
+	options.Loader.Load(options.Source...)
+	snap, _ := options.Loader.Snapshot()
+	vals, _ := options.Reader.Values(snap.ChangeSet)
+
+	c := &config{
+		exit: make(chan bool),
+		opts: options,
+		snap: snap,
+		vals: vals,
+	}
+
+	go c.run()
+
+	return c
+}
+
+func (c *config) run() {
+	watch := func(w loader.Watcher) error {
+		for {
+			// get changeset
+			snap, err := w.Next()
+			if err != nil {
+				return err
+			}
+
+			c.Lock()
+
+			// save
+			c.snap = snap
+
+			// set values
+			c.vals, _ = c.opts.Reader.Values(snap.ChangeSet)
+
+			c.Unlock()
+		}
+	}
+
+	for {
+		w, err := c.opts.Loader.Watch()
+		if err != nil {
+			time.Sleep(time.Second)
+			continue
+		}
+
+		done := make(chan bool)
+
+		// the stop watch func
+		go func() {
+			select {
+			case <-done:
+			case <-c.exit:
+			}
+			w.Stop()
+		}()
+
+		// block watch
+		if err := watch(w); err != nil {
+			// do something better
+			time.Sleep(time.Second)
+		}
+
+		// close done chan
+		close(done)
+
+		// if the config is closed exit
+		select {
+		case <-c.exit:
+			return
+		default:
+		}
+	}
+}
+
+func (c *config) Map() map[string]interface{} {
+	c.RLock()
+	defer c.RUnlock()
+	return c.vals.Map()
+}
+
+func (c *config) Scan(v interface{}) error {
+	c.RLock()
+	defer c.RUnlock()
+	return c.vals.Scan(v)
+}
+
+// sync loads all the sources, calls the parser and updates the config
+func (c *config) Sync() error {
+	if err := c.opts.Loader.Sync(); err != nil {
+		return err
+	}
+
+	snap, err := c.opts.Loader.Snapshot()
+	if err != nil {
+		return err
+	}
+
+	c.Lock()
+	defer c.Unlock()
+
+	c.snap = snap
+	vals, err := c.opts.Reader.Values(snap.ChangeSet)
+	if err != nil {
+		return err
+	}
+	c.vals = vals
+
+	return nil
+}
+
+func (c *config) Close() error {
+	select {
+	case <-c.exit:
+		return nil
+	default:
+		close(c.exit)
+	}
+	return nil
+}
+
+func (c *config) Get(path ...string) reader.Value {
+	c.RLock()
+	defer c.RUnlock()
+
+	// did sync actually work?
+	if c.vals != nil {
+		return c.vals.Get(path...)
+	}
+
+	// no value
+	return newValue()
+}
+
+func (c *config) Bytes() []byte {
+	c.RLock()
+	defer c.RUnlock()
+
+	if c.vals == nil {
+		return []byte{}
+	}
+
+	return c.vals.Bytes()
+}
+
+func (c *config) Load(sources ...source.Source) error {
+	if err := c.opts.Loader.Load(sources...); err != nil {
+		return err
+	}
+
+	snap, err := c.opts.Loader.Snapshot()
+	if err != nil {
+		return err
+	}
+
+	c.Lock()
+	defer c.Unlock()
+
+	c.snap = snap
+	vals, err := c.opts.Reader.Values(snap.ChangeSet)
+	if err != nil {
+		return err
+	}
+	c.vals = vals
+
+	return nil
+}
+
+func (c *config) Watch(path ...string) (Watcher, error) {
+	value := c.Get(path...)
+
+	w, err := c.opts.Loader.Watch(path...)
+	if err != nil {
+		return nil, err
+	}
+
+	return &watcher{
+		lw:    w,
+		rd:    c.opts.Reader,
+		path:  path,
+		value: value,
+	}, nil
+}
+
+func (c *config) String() string {
+	return "config"
+}
+
+func (w *watcher) Next() (reader.Value, error) {
+	for {
+		s, err := w.lw.Next()
+		if err != nil {
+			return nil, err
+		}
+
+		// only process changes
+		if bytes.Equal(w.value.Bytes(), s.ChangeSet.Data) {
+			continue
+		}
+
+		v, err := w.rd.Values(s.ChangeSet)
+		if err != nil {
+			return nil, err
+		}
+
+		w.value = v.Get()
+		return w.value, nil
+	}
+}
+
+func (w *watcher) Stop() error {
+	return w.lw.Stop()
+}

config/default_test.go

@@ -0,0 +1,101 @@
+package config
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/micro/go-micro/config/source/file"
+)
+
+func createFileForTest(t *testing.T) *os.File {
+	data := []byte(`{"foo": "bar"}`)
+	path := filepath.Join(os.TempDir(), fmt.Sprintf("file.%d", time.Now().UnixNano()))
+	fh, err := os.Create(path)
+	if err != nil {
+		t.Error(err)
+	}
+	_, err = fh.Write(data)
+	if err != nil {
+		t.Error(err)
+	}
+
+	return fh
+}
+
+func TestLoadWithGoodFile(t *testing.T) {
+	fh := createFileForTest(t)
+	path := fh.Name()
+	defer func() {
+		fh.Close()
+		os.Remove(path)
+	}()
+
+	// Create new config
+	conf := NewConfig()
+	// Load file source
+	if err := conf.Load(file.NewSource(
+		file.WithPath(path),
+	)); err != nil {
+		t.Fatalf("Expected no error but got %v", err)
+	}
+}
+
+func TestLoadWithInvalidFile(t *testing.T) {
+	fh := createFileForTest(t)
+	path := fh.Name()
+	defer func() {
+		fh.Close()
+		os.Remove(path)
+	}()
+
+	// Create new config
+	conf := NewConfig()
+	// Load file source
+	err := conf.Load(file.NewSource(
+		file.WithPath(path),
+		file.WithPath("/i/do/not/exists.json"),
+	))
+
+	if err == nil {
+		t.Fatal("Expected error but none !")
+	}
+	if !strings.Contains(fmt.Sprintf("%v", err), "/i/do/not/exists.json") {
+		t.Fatalf("Expected error to contain the unexisting file but got %v", err)
+	}
+}
+
+func TestConsul(t *testing.T) {
+	/*consulSource := consul.NewSource(
+		// optionally specify consul address; default to localhost:8500
+		consul.WithAddress("131.150.38.111:8500"),
+		// optionally specify prefix; defaults to /micro/config
+		consul.WithPrefix("/project"),
+		// optionally strip the provided prefix from the keys, defaults to false
+		consul.StripPrefix(true),
+		consul.WithDatacenter("dc1"),
+		consul.WithToken("xxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx"),
+	)
+
+	// Create new config
+	conf := NewConfig()
+
+	// Load file source
+	err := conf.Load(consulSource)
+	if err != nil {
+		t.Error(err)
+		return
+	}
+
+	m := conf.Map()
+	t.Log("m: ", m)
+
+	v := conf.Get("project", "dc111", "port")
+
+	t.Log("v: ", v.Int(13))*/
+
+	t.Log("OK")
+}

config/encoder/encoder.go

@@ -0,0 +1,8 @@
+// Package encoder handles source encoding formats
+package encoder
+
+type Encoder interface {
+	Encode(interface{}) ([]byte, error)
+	Decode([]byte, interface{}) error
+	String() string
+}

config/encoder/hcl/hcl.go

@@ -0,0 +1,26 @@
+package hcl
+
+import (
+	"encoding/json"
+
+	"github.com/hashicorp/hcl"
+	"github.com/micro/go-micro/config/encoder"
+)
+
+type hclEncoder struct{}
+
+func (h hclEncoder) Encode(v interface{}) ([]byte, error) {
+	return json.Marshal(v)
+}
+
+func (h hclEncoder) Decode(d []byte, v interface{}) error {
+	return hcl.Unmarshal(d, v)
+}
+
+func (h hclEncoder) String() string {
+	return "hcl"
+}
+
+func NewEncoder() encoder.Encoder {
+	return hclEncoder{}
+}

config/encoder/json/json.go

@@ -0,0 +1,25 @@
+package json
+
+import (
+	"encoding/json"
+
+	"github.com/micro/go-micro/config/encoder"
+)
+
+type jsonEncoder struct{}
+
+func (j jsonEncoder) Encode(v interface{}) ([]byte, error) {
+	return json.Marshal(v)
+}
+
+func (j jsonEncoder) Decode(d []byte, v interface{}) error {
+	return json.Unmarshal(d, v)
+}
+
+func (j jsonEncoder) String() string {
+	return "json"
+}
+
+func NewEncoder() encoder.Encoder {
+	return jsonEncoder{}
+}

config/encoder/toml/toml.go

@@ -0,0 +1,32 @@
+package toml
+
+import (
+	"bytes"
+
+	"github.com/BurntSushi/toml"
+	"github.com/micro/go-micro/config/encoder"
+)
+
+type tomlEncoder struct{}
+
+func (t tomlEncoder) Encode(v interface{}) ([]byte, error) {
+	b := bytes.NewBuffer(nil)
+	defer b.Reset()
+	err := toml.NewEncoder(b).Encode(v)
+	if err != nil {
+		return nil, err
+	}
+	return b.Bytes(), nil
+}
+
+func (t tomlEncoder) Decode(d []byte, v interface{}) error {
+	return toml.Unmarshal(d, v)
+}
+
+func (t tomlEncoder) String() string {
+	return "toml"
+}
+
+func NewEncoder() encoder.Encoder {
+	return tomlEncoder{}
+}

config/encoder/xml/xml.go

@@ -0,0 +1,25 @@
+package xml
+
+import (
+	"encoding/xml"
+
+	"github.com/micro/go-micro/config/encoder"
+)
+
+type xmlEncoder struct{}
+
+func (x xmlEncoder) Encode(v interface{}) ([]byte, error) {
+	return xml.Marshal(v)
+}
+
+func (x xmlEncoder) Decode(d []byte, v interface{}) error {
+	return xml.Unmarshal(d, v)
+}
+
+func (x xmlEncoder) String() string {
+	return "xml"
+}
+
+func NewEncoder() encoder.Encoder {
+	return xmlEncoder{}
+}

config/encoder/yaml/yaml.go

@@ -0,0 +1,24 @@
+package yaml
+
+import (
+	"github.com/ghodss/yaml"
+	"github.com/micro/go-micro/config/encoder"
+)
+
+type yamlEncoder struct{}
+
+func (y yamlEncoder) Encode(v interface{}) ([]byte, error) {
+	return yaml.Marshal(v)
+}
+
+func (y yamlEncoder) Decode(d []byte, v interface{}) error {
+	return yaml.Unmarshal(d, v)
+}
+
+func (y yamlEncoder) String() string {
+	return "yaml"
+}
+
+func NewEncoder() encoder.Encoder {
+	return yamlEncoder{}
+}

config/issue18_test.go

@@ -0,0 +1,60 @@
+package config
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/micro/go-micro/config/source/env"
+	"github.com/micro/go-micro/config/source/file"
+)
+
+func createFileForIssue18(t *testing.T, content string) *os.File {
+	data := []byte(content)
+	path := filepath.Join(os.TempDir(), fmt.Sprintf("file.%d", time.Now().UnixNano()))
+	fh, err := os.Create(path)
+	if err != nil {
+		t.Error(err)
+	}
+	_, err = fh.Write(data)
+	if err != nil {
+		t.Error(err)
+	}
+
+	return fh
+}
+
+func TestIssue18(t *testing.T) {
+	fh := createFileForIssue18(t, `{
+  "amqp": {
+    "host": "rabbit.platform",
+    "port": 80
+  },
+  "handler": {
+    "exchange": "springCloudBus"
+  }
+}`)
+	path := fh.Name()
+	defer func() {
+		fh.Close()
+		os.Remove(path)
+	}()
+	os.Setenv("AMQP_HOST", "rabbit.testing.com")
+
+	conf := NewConfig()
+	conf.Load(
+		file.NewSource(
+			file.WithPath(path),
+		),
+		env.NewSource(),
+	)
+
+	actualHost := conf.Get("amqp", "host").String("backup")
+	if actualHost != "rabbit.testing.com" {
+		t.Fatalf("Expected %v but got %v",
+			"rabbit.testing.com",
+			actualHost)
+	}
+}

config/loader/loader.go

@@ -0,0 +1,63 @@
+// package loader manages loading from multiple sources
+package loader
+
+import (
+	"context"
+
+	"github.com/micro/go-micro/config/reader"
+	"github.com/micro/go-micro/config/source"
+)
+
+// Loader manages loading sources
+type Loader interface {
+	// Stop the loader
+	Close() error
+	// Load the sources
+	Load(...source.Source) error
+	// A Snapshot of loaded config
+	Snapshot() (*Snapshot, error)
+	// Force sync of sources
+	Sync() error
+	// Watch for changes
+	Watch(...string) (Watcher, error)
+	// Name of loader
+	String() string
+}
+
+// Watcher lets you watch sources and returns a merged ChangeSet
+type Watcher interface {
+	// First call to next may return the current Snapshot
+	// If you are watching a path then only the data from
+	// that path is returned.
+	Next() (*Snapshot, error)
+	// Stop watching for changes
+	Stop() error
+}
+
+// Snapshot is a merged ChangeSet
+type Snapshot struct {
+	// The merged ChangeSet
+	ChangeSet *source.ChangeSet
+	// Deterministic and comparable version of the snapshot
+	Version string
+}
+
+type Options struct {
+	Reader reader.Reader
+	Source []source.Source
+
+	// for alternative data
+	Context context.Context
+}
+
+type Option func(o *Options)
+
+// Copy snapshot
+func Copy(s *Snapshot) *Snapshot {
+	cs := *(s.ChangeSet)
+
+	return &Snapshot{
+		ChangeSet: &cs,
+		Version:   s.Version,
+	}
+}

config/loader/memory/memory.go

@@ -0,0 +1,415 @@
+package memory
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/micro/go-micro/config/loader"
+	"github.com/micro/go-micro/config/reader"
+	"github.com/micro/go-micro/config/reader/json"
+	"github.com/micro/go-micro/config/source"
+)
+
+type memory struct {
+	exit chan bool
+	opts loader.Options
+
+	sync.RWMutex
+	// the current snapshot
+	snap *loader.Snapshot
+	// the current values
+	vals reader.Values
+	// all the sets
+	sets []*source.ChangeSet
+	// all the sources
+	sources []source.Source
+
+	idx      int
+	watchers map[int]*watcher
+}
+
+type watcher struct {
+	exit    chan bool
+	path    []string
+	value   reader.Value
+	reader  reader.Reader
+	updates chan reader.Value
+}
+
+func (m *memory) watch(idx int, s source.Source) {
+	m.Lock()
+	m.sets = append(m.sets, &source.ChangeSet{Source: s.String()})
+	m.Unlock()
+
+	// watches a source for changes
+	watch := func(idx int, s source.Watcher) error {
+		for {
+			// get changeset
+			cs, err := s.Next()
+			if err != nil {
+				return err
+			}
+
+			m.Lock()
+
+			// save
+			m.sets[idx] = cs
+
+			// merge sets
+			set, err := m.opts.Reader.Merge(m.sets...)
+			if err != nil {
+				m.Unlock()
+				return err
+			}
+
+			// set values
+			m.vals, _ = m.opts.Reader.Values(set)
+			m.snap = &loader.Snapshot{
+				ChangeSet: set,
+				Version:   fmt.Sprintf("%d", time.Now().Unix()),
+			}
+			m.Unlock()
+
+			// send watch updates
+			m.update()
+		}
+	}
+
+	for {
+		// watch the source
+		w, err := s.Watch()
+		if err != nil {
+			time.Sleep(time.Second)
+			continue
+		}
+
+		done := make(chan bool)
+
+		// the stop watch func
+		go func() {
+			select {
+			case <-done:
+			case <-m.exit:
+			}
+			w.Stop()
+		}()
+
+		// block watch
+		if err := watch(idx, w); err != nil {
+			// do something better
+			time.Sleep(time.Second)
+		}
+
+		// close done chan
+		close(done)
+
+		// if the config is closed exit
+		select {
+		case <-m.exit:
+			return
+		default:
+		}
+	}
+}
+
+func (m *memory) loaded() bool {
+	var loaded bool
+	m.RLock()
+	if m.vals != nil {
+		loaded = true
+	}
+	m.RUnlock()
+	return loaded
+}
+
+// reload reads the sets and creates new values
+func (m *memory) reload() error {
+	m.Lock()
+
+	// merge sets
+	set, err := m.opts.Reader.Merge(m.sets...)
+	if err != nil {
+		m.Unlock()
+		return err
+	}
+
+	// set values
+	m.vals, _ = m.opts.Reader.Values(set)
+	m.snap = &loader.Snapshot{
+		ChangeSet: set,
+		Version:   fmt.Sprintf("%d", time.Now().Unix()),
+	}
+
+	m.Unlock()
+
+	// update watchers
+	m.update()
+
+	return nil
+}
+
+func (m *memory) update() {
+	var watchers []*watcher
+
+	m.RLock()
+	for _, w := range m.watchers {
+		watchers = append(watchers, w)
+	}
+	m.RUnlock()
+
+	for _, w := range watchers {
+		select {
+		case w.updates <- m.vals.Get(w.path...):
+		default:
+		}
+	}
+}
+
+// Snapshot returns a snapshot of the current loaded config
+func (m *memory) Snapshot() (*loader.Snapshot, error) {
+	if m.loaded() {
+		m.RLock()
+		snap := loader.Copy(m.snap)
+		m.RUnlock()
+		return snap, nil
+	}
+
+	// not loaded, sync
+	if err := m.Sync(); err != nil {
+		return nil, err
+	}
+
+	// make copy
+	m.RLock()
+	snap := loader.Copy(m.snap)
+	m.RUnlock()
+
+	return snap, nil
+}
+
+// Sync loads all the sources, calls the parser and updates the config
+func (m *memory) Sync() error {
+	var sets []*source.ChangeSet
+
+	m.Lock()
+
+	// read the source
+	var gerr []string
+
+	for _, source := range m.sources {
+		ch, err := source.Read()
+		if err != nil {
+			gerr = append(gerr, err.Error())
+			continue
+		}
+		sets = append(sets, ch)
+	}
+
+	// merge sets
+	set, err := m.opts.Reader.Merge(sets...)
+	if err != nil {
+		m.Unlock()
+		return err
+	}
+
+	// set values
+	vals, err := m.opts.Reader.Values(set)
+	if err != nil {
+		m.Unlock()
+		return err
+	}
+	m.vals = vals
+	m.snap = &loader.Snapshot{
+		ChangeSet: set,
+		Version:   fmt.Sprintf("%d", time.Now().Unix()),
+	}
+
+	m.Unlock()
+
+	// update watchers
+	m.update()
+
+	if len(gerr) > 0 {
+		return fmt.Errorf("source loading errors: %s", strings.Join(gerr, "\n"))
+	}
+
+	return nil
+}
+
+func (m *memory) Close() error {
+	select {
+	case <-m.exit:
+		return nil
+	default:
+		close(m.exit)
+	}
+	return nil
+}
+
+func (m *memory) Get(path ...string) (reader.Value, error) {
+	if !m.loaded() {
+		if err := m.Sync(); err != nil {
+			return nil, err
+		}
+	}
+
+	m.Lock()
+	defer m.Unlock()
+
+	// did sync actually work?
+	if m.vals != nil {
+		return m.vals.Get(path...), nil
+	}
+
+	// assuming vals is nil
+	// create new vals
+
+	ch := m.snap.ChangeSet
+
+	// we are truly screwed, trying to load in a hacked way
+	v, err := m.opts.Reader.Values(ch)
+	if err != nil {
+		return nil, err
+	}
+
+	// lets set it just because
+	m.vals = v
+
+	if m.vals != nil {
+		return m.vals.Get(path...), nil
+	}
+
+	// ok we're going hardcore now
+	return nil, errors.New("no values")
+}
+
+func (m *memory) Load(sources ...source.Source) error {
+	var gerrors []string
+
+	for _, source := range sources {
+		set, err := source.Read()
+		if err != nil {
+			gerrors = append(gerrors,
+				fmt.Sprintf("error loading source %s: %v",
+					source,
+					err))
+			// continue processing
+			continue
+		}
+		m.Lock()
+		m.sources = append(m.sources, source)
+		m.sets = append(m.sets, set)
+		idx := len(m.sets) - 1
+		m.Unlock()
+		go m.watch(idx, source)
+	}
+
+	if err := m.reload(); err != nil {
+		gerrors = append(gerrors, err.Error())
+	}
+
+	// Return errors
+	if len(gerrors) != 0 {
+		return errors.New(strings.Join(gerrors, "\n"))
+	}
+	return nil
+}
+
+func (m *memory) Watch(path ...string) (loader.Watcher, error) {
+	value, err := m.Get(path...)
+	if err != nil {
+		return nil, err
+	}
+
+	m.Lock()
+
+	w := &watcher{
+		exit:    make(chan bool),
+		path:    path,
+		value:   value,
+		reader:  m.opts.Reader,
+		updates: make(chan reader.Value, 1),
+	}
+
+	id := m.idx
+	m.watchers[id] = w
+	m.idx++
+
+	m.Unlock()
+
+	go func() {
+		<-w.exit
+		m.Lock()
+		delete(m.watchers, id)
+		m.Unlock()
+	}()
+
+	return w, nil
+}
+
+func (m *memory) String() string {
+	return "memory"
+}
+
+func (w *watcher) Next() (*loader.Snapshot, error) {
+	for {
+		select {
+		case <-w.exit:
+			return nil, errors.New("watcher stopped")
+		case v := <-w.updates:
+			if bytes.Equal(w.value.Bytes(), v.Bytes()) {
+				continue
+			}
+			w.value = v
+
+			cs := &source.ChangeSet{
+				Data:      v.Bytes(),
+				Format:    w.reader.String(),
+				Source:    "memory",
+				Timestamp: time.Now(),
+			}
+			cs.Sum()
+
+			return &loader.Snapshot{
+				ChangeSet: cs,
+				Version:   fmt.Sprintf("%d", time.Now().Unix()),
+			}, nil
+		}
+	}
+}
+
+func (w *watcher) Stop() error {
+	select {
+	case <-w.exit:
+	default:
+		close(w.exit)
+	}
+	return nil
+}
+
+func NewLoader(opts ...loader.Option) loader.Loader {
+	options := loader.Options{
+		Reader: json.NewReader(),
+	}
+
+	for _, o := range opts {
+		o(&options)
+	}
+
+	m := &memory{
+		exit:     make(chan bool),
+		opts:     options,
+		watchers: make(map[int]*watcher),
+		sources:  options.Source,
+	}
+
+	for i, s := range options.Source {
+		go m.watch(i, s)
+	}
+
+	return m
+}

config/loader/memory/options.go

@@ -0,0 +1,21 @@
+package memory
+
+import (
+	"github.com/micro/go-micro/config/loader"
+	"github.com/micro/go-micro/config/reader"
+	"github.com/micro/go-micro/config/source"
+)
+
+// WithSource appends a source to list of sources
+func WithSource(s source.Source) loader.Option {
+	return func(o *loader.Options) {
+		o.Source = append(o.Source, s)
+	}
+}
+
+// WithReader sets the config reader
+func WithReader(r reader.Reader) loader.Option {
+	return func(o *loader.Options) {
+		o.Reader = r
+	}
+}

config/options.go

@@ -0,0 +1,28 @@
+package config
+
+import (
+	"github.com/micro/go-micro/config/loader"
+	"github.com/micro/go-micro/config/reader"
+	"github.com/micro/go-micro/config/source"
+)
+
+// WithLoader sets the loader for manager config
+func WithLoader(l loader.Loader) Option {
+	return func(o *Options) {
+		o.Loader = l
+	}
+}
+
+// WithSource appends a source to list of sources
+func WithSource(s source.Source) Option {
+	return func(o *Options) {
+		o.Source = append(o.Source, s)
+	}
+}
+
+// WithReader sets the config reader
+func WithReader(r reader.Reader) Option {
+	return func(o *Options) {
+		o.Reader = r
+	}
+}

config/reader/json/json.go

@@ -0,0 +1,83 @@
+package json
+
+import (
+	"errors"
+	"time"
+
+	"github.com/imdario/mergo"
+	"github.com/micro/go-micro/config/encoder"
+	"github.com/micro/go-micro/config/encoder/json"
+	"github.com/micro/go-micro/config/reader"
+	"github.com/micro/go-micro/config/source"
+)
+
+type jsonReader struct {
+	opts reader.Options
+	json encoder.Encoder
+}
+
+func (j *jsonReader) Merge(changes ...*source.ChangeSet) (*source.ChangeSet, error) {
+	var merged map[string]interface{}
+
+	for _, m := range changes {
+		if m == nil {
+			continue
+		}
+
+		if len(m.Data) == 0 {
+			continue
+		}
+
+		codec, ok := j.opts.Encoding[m.Format]
+		if !ok {
+			// fallback
+			codec = j.json
+		}
+
+		var data map[string]interface{}
+		if err := codec.Decode(m.Data, &data); err != nil {
+			return nil, err
+		}
+		if err := mergo.Map(&merged, data, mergo.WithOverride); err != nil {
+			return nil, err
+		}
+	}
+
+	b, err := j.json.Encode(merged)
+	if err != nil {
+		return nil, err
+	}
+
+	cs := &source.ChangeSet{
+		Timestamp: time.Now(),
+		Data:      b,
+		Source:    "json",
+		Format:    j.json.String(),
+	}
+	cs.Checksum = cs.Sum()
+
+	return cs, nil
+}
+
+func (j *jsonReader) Values(ch *source.ChangeSet) (reader.Values, error) {
+	if ch == nil {
+		return nil, errors.New("changeset is nil")
+	}
+	if ch.Format != "json" {
+		return nil, errors.New("unsupported format")
+	}
+	return newValues(ch)
+}
+
+func (j *jsonReader) String() string {
+	return "json"
+}
+
+// NewReader creates a json reader
+func NewReader(opts ...reader.Option) reader.Reader {
+	options := reader.NewOptions(opts...)
+	return &jsonReader{
+		json: json.NewEncoder(),
+		opts: options,
+	}
+}

config/reader/json/json_test.go

@@ -0,0 +1,43 @@
+package json
+
+import (
+	"testing"
+
+	"github.com/micro/go-micro/config/source"
+)
+
+func TestReader(t *testing.T) {
+	data := []byte(`{"foo": "bar", "baz": {"bar": "cat"}}`)
+
+	testData := []struct {
+		path  []string
+		value string
+	}{
+		{
+			[]string{"foo"},
+			"bar",
+		},
+		{
+			[]string{"baz", "bar"},
+			"cat",
+		},
+	}
+
+	r := NewReader()
+
+	c, err := r.Merge(&source.ChangeSet{Data: data}, &source.ChangeSet{})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	values, err := r.Values(c)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	for _, test := range testData {
+		if v := values.Get(test.path...).String(""); v != test.value {
+			t.Fatalf("Expected %s got %s for path %v", test.value, v, test.path)
+		}
+	}
+}

config/reader/json/values.go

@@ -0,0 +1,208 @@
+package json
+
+import (
+	"encoding/json"
+	"fmt"
+	"strconv"
+	"strings"
+	"time"
+
+	simple "github.com/bitly/go-simplejson"
+	"github.com/micro/go-micro/config/reader"
+	"github.com/micro/go-micro/config/source"
+)
+
+type jsonValues struct {
+	ch *source.ChangeSet
+	sj *simple.Json
+}
+
+type jsonValue struct {
+	*simple.Json
+}
+
+func newValues(ch *source.ChangeSet) (reader.Values, error) {
+	sj := simple.New()
+	data, _ := reader.ReplaceEnvVars(ch.Data)
+	if err := sj.UnmarshalJSON(data); err != nil {
+		sj.SetPath(nil, string(ch.Data))
+	}
+	return &jsonValues{ch, sj}, nil
+}
+
+func newValue(s *simple.Json) reader.Value {
+	if s == nil {
+		s = simple.New()
+	}
+	return &jsonValue{s}
+}
+
+func (j *jsonValues) Get(path ...string) reader.Value {
+	return &jsonValue{j.sj.GetPath(path...)}
+}
+
+func (j *jsonValues) Del(path ...string) {
+	// delete the tree?
+	if len(path) == 0 {
+		j.sj = simple.New()
+		return
+	}
+
+	if len(path) == 1 {
+		j.sj.Del(path[0])
+		return
+	}
+
+	vals := j.sj.GetPath(path[:len(path)-1]...)
+	vals.Del(path[len(path)-1])
+	j.sj.SetPath(path[:len(path)-1], vals.Interface())
+	return
+}
+
+func (j *jsonValues) Set(val interface{}, path ...string) {
+	j.sj.SetPath(path, val)
+}
+
+func (j *jsonValues) Bytes() []byte {
+	b, _ := j.sj.MarshalJSON()
+	return b
+}
+
+func (j *jsonValues) Map() map[string]interface{} {
+	m, _ := j.sj.Map()
+	return m
+}
+
+func (j *jsonValues) Scan(v interface{}) error {
+	b, err := j.sj.MarshalJSON()
+	if err != nil {
+		return err
+	}
+	return json.Unmarshal(b, v)
+}
+
+func (j *jsonValues) String() string {
+	return "json"
+}
+
+func (j *jsonValue) Bool(def bool) bool {
+	b, err := j.Json.Bool()
+	if err == nil {
+		return b
+	}
+
+	str, ok := j.Interface().(string)
+	if !ok {
+		return def
+	}
+
+	b, err = strconv.ParseBool(str)
+	if err != nil {
+		return def
+	}
+
+	return b
+}
+
+func (j *jsonValue) Int(def int) int {
+	i, err := j.Json.Int()
+	if err == nil {
+		return i
+	}
+
+	str, ok := j.Interface().(string)
+	if !ok {
+		return def
+	}
+
+	i, err = strconv.Atoi(str)
+	if err != nil {
+		return def
+	}
+
+	return i
+}
+
+func (j *jsonValue) String(def string) string {
+	return j.Json.MustString(def)
+}
+
+func (j *jsonValue) Float64(def float64) float64 {
+	f, err := j.Json.Float64()
+	if err == nil {
+		return f
+	}
+
+	str, ok := j.Interface().(string)
+	if !ok {
+		return def
+	}
+
+	f, err = strconv.ParseFloat(str, 64)
+	if err != nil {
+		return def
+	}
+
+	return f
+}
+
+func (j *jsonValue) Duration(def time.Duration) time.Duration {
+	v, err := j.Json.String()
+	if err != nil {
+		return def
+	}
+
+	value, err := time.ParseDuration(v)
+	if err != nil {
+		return def
+	}
+
+	return value
+}
+
+func (j *jsonValue) StringSlice(def []string) []string {
+	v, err := j.Json.String()
+	if err == nil {
+		sl := strings.Split(v, ",")
+		if len(sl) > 1 {
+			return sl
+		}
+	}
+	return j.Json.MustStringArray(def)
+}
+
+func (j *jsonValue) StringMap(def map[string]string) map[string]string {
+	m, err := j.Json.Map()
+	if err != nil {
+		return def
+	}
+
+	res := map[string]string{}
+
+	for k, v := range m {
+		res[k] = fmt.Sprintf("%v", v)
+	}
+
+	return res
+}
+
+func (j *jsonValue) Scan(v interface{}) error {
+	b, err := j.Json.MarshalJSON()
+	if err != nil {
+		return err
+	}
+	return json.Unmarshal(b, v)
+}
+
+func (j *jsonValue) Bytes() []byte {
+	b, err := j.Json.Bytes()
+	if err != nil {
+		// try return marshalled
+		b, err = j.Json.MarshalJSON()
+		if err != nil {
+			return []byte{}
+		}
+		return b
+	}
+	return b
+}

config/reader/json/values_test.go

@@ -0,0 +1,39 @@
+package json
+
+import (
+	"testing"
+
+	"github.com/micro/go-micro/config/source"
+)
+
+func TestValues(t *testing.T) {
+	data := []byte(`{"foo": "bar", "baz": {"bar": "cat"}}`)
+
+	testData := []struct {
+		path  []string
+		value string
+	}{
+		{
+			[]string{"foo"},
+			"bar",
+		},
+		{
+			[]string{"baz", "bar"},
+			"cat",
+		},
+	}
+
+	values, err := newValues(&source.ChangeSet{
+		Data: data,
+	})
+
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	for _, test := range testData {
+		if v := values.Get(test.path...).String(""); v != test.value {
+			t.Fatalf("Expected %s got %s for path %v", test.value, v, test.path)
+		}
+	}
+}

config/reader/options.go

@@ -0,0 +1,42 @@
+package reader
+
+import (
+	"github.com/micro/go-micro/config/encoder"
+	"github.com/micro/go-micro/config/encoder/hcl"
+	"github.com/micro/go-micro/config/encoder/json"
+	"github.com/micro/go-micro/config/encoder/toml"
+	"github.com/micro/go-micro/config/encoder/xml"
+	"github.com/micro/go-micro/config/encoder/yaml"
+)
+
+type Options struct {
+	Encoding map[string]encoder.Encoder
+}
+
+type Option func(o *Options)
+
+func NewOptions(opts ...Option) Options {
+	options := Options{
+		Encoding: map[string]encoder.Encoder{
+			"json": json.NewEncoder(),
+			"yaml": yaml.NewEncoder(),
+			"toml": toml.NewEncoder(),
+			"xml":  xml.NewEncoder(),
+			"hcl":  hcl.NewEncoder(),
+			"yml":  yaml.NewEncoder(),
+		},
+	}
+	for _, o := range opts {
+		o(&options)
+	}
+	return options
+}
+
+func WithEncoder(e encoder.Encoder) Option {
+	return func(o *Options) {
+		if o.Encoding == nil {
+			o.Encoding = make(map[string]encoder.Encoder)
+		}
+		o.Encoding[e.String()] = e
+	}
+}

config/reader/preprocessor.go

@@ -0,0 +1,23 @@
+package reader
+
+import (
+	"os"
+	"regexp"
+)
+
+func ReplaceEnvVars(raw []byte) ([]byte, error) {
+	re := regexp.MustCompile(`\$\{([A-Za-z0-9_]+)\}`)
+	if re.Match(raw) {
+		dataS := string(raw)
+		res := re.ReplaceAllStringFunc(dataS, replaceEnvVars)
+		return []byte(res), nil
+	} else {
+		return raw, nil
+	}
+}
+
+func replaceEnvVars(element string) string {
+	v := element[2 : len(element)-1]
+	el := os.Getenv(v)
+	return el
+}

config/reader/preprocessor_test.go

@@ -0,0 +1,73 @@
+package reader
+
+import (
+	"os"
+	"strings"
+	"testing"
+)
+
+func TestReplaceEnvVars(t *testing.T) {
+	os.Setenv("myBar", "cat")
+	os.Setenv("MYBAR", "cat")
+	os.Setenv("my_Bar", "cat")
+	os.Setenv("myBar_", "cat")
+
+	testData := []struct {
+		expected string
+		data     []byte
+	}{
+		// Right use cases
+		{
+			`{"foo": "bar", "baz": {"bar": "cat"}}`,
+			[]byte(`{"foo": "bar", "baz": {"bar": "${myBar}"}}`),
+		},
+		{
+			`{"foo": "bar", "baz": {"bar": "cat"}}`,
+			[]byte(`{"foo": "bar", "baz": {"bar": "${MYBAR}"}}`),
+		},
+		{
+			`{"foo": "bar", "baz": {"bar": "cat"}}`,
+			[]byte(`{"foo": "bar", "baz": {"bar": "${my_Bar}"}}`),
+		},
+		{
+			`{"foo": "bar", "baz": {"bar": "cat"}}`,
+			[]byte(`{"foo": "bar", "baz": {"bar": "${myBar_}"}}`),
+		},
+		// Wrong use cases
+		{
+			`{"foo": "bar", "baz": {"bar": "${myBar-}"}}`,
+			[]byte(`{"foo": "bar", "baz": {"bar": "${myBar-}"}}`),
+		},
+		{
+			`{"foo": "bar", "baz": {"bar": "${}"}}`,
+			[]byte(`{"foo": "bar", "baz": {"bar": "${}"}}`),
+		},
+		{
+			`{"foo": "bar", "baz": {"bar": "$sss}"}}`,
+			[]byte(`{"foo": "bar", "baz": {"bar": "$sss}"}}`),
+		},
+		{
+			`{"foo": "bar", "baz": {"bar": "${sss"}}`,
+			[]byte(`{"foo": "bar", "baz": {"bar": "${sss"}}`),
+		},
+		{
+			`{"foo": "bar", "baz": {"bar": "{something}"}}`,
+			[]byte(`{"foo": "bar", "baz": {"bar": "{something}"}}`),
+		},
+		// Use cases without replace env vars
+		{
+			`{"foo": "bar", "baz": {"bar": "cat"}}`,
+			[]byte(`{"foo": "bar", "baz": {"bar": "cat"}}`),
+		},
+	}
+
+	for _, test := range testData {
+		res, err := ReplaceEnvVars(test.data)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if strings.Compare(test.expected, string(res)) != 0 {
+			t.Fatalf("Expected %s got %s", test.expected, res)
+		}
+	}
+}

config/reader/reader.go

@@ -0,0 +1,36 @@
+// Package reader parses change sets and provides config values
+package reader
+
+import (
+	"time"
+
+	"github.com/micro/go-micro/config/source"
+)
+
+// Reader is an interface for merging changesets
+type Reader interface {
+	Merge(...*source.ChangeSet) (*source.ChangeSet, error)
+	Values(*source.ChangeSet) (Values, error)
+	String() string
+}
+
+// Values is returned by the reader
+type Values interface {
+	Bytes() []byte
+	Get(path ...string) Value
+	Map() map[string]interface{}
+	Scan(v interface{}) error
+}
+
+// Value represents a value of any type
+type Value interface {
+	Bool(def bool) bool
+	Int(def int) int
+	String(def string) string
+	Float64(def float64) float64
+	Duration(def time.Duration) time.Duration
+	StringSlice(def []string) []string
+	StringMap(def map[string]string) map[string]string
+	Scan(val interface{}) error
+	Bytes() []byte
+}

config/source/changeset.go

@@ -0,0 +1,13 @@
+package source
+
+import (
+	"crypto/md5"
+	"fmt"
+)
+
+// Sum returns the md5 checksum of the ChangeSet data
+func (c *ChangeSet) Sum() string {
+	h := md5.New()
+	h.Write(c.Data)
+	return fmt.Sprintf("%x", h.Sum(nil))
+}

config/source/cli/README.md

@@ -0,0 +1,71 @@
+# cli Source
+
+The cli source reads config from parsed flags via a cli.Context.
+
+## Format
+
+We expect the use of the `micro/cli` package. Upper case flags will be lower cased. Dashes will be used as delimiters for nesting.
+
+### Example
+
+```go
+micro.Flags(
+    cli.StringFlag{
+        Name: "database-address",
+        Value: "127.0.0.1",
+        Usage: "the db address",
+    },
+    cli.IntFlag{
+        Name: "database-port",
+        Value: 3306,
+        Usage: "the db port",
+    },
+)
+```
+
+Becomes
+
+```json
+{
+    "database": {
+        "address": "127.0.0.1",
+        "port": 3306
+    }
+}
+```
+
+## New and Load Source
+
+Because a cli.Context is needed to retrieve the flags and their values, it is recommended to build your source from within a cli.Action.
+
+```go
+
+func main() {
+    // New Service
+    service := micro.NewService(
+        micro.Name("example"),
+        micro.Flags(
+            cli.StringFlag{
+                Name: "database-address",
+                Value: "127.0.0.1",
+                Usage: "the db address",
+            },
+        ),
+    )
+
+    var clisrc source.Source
+
+    service.Init(
+        micro.Action(func(c *cli.Context) {
+            clisrc = cli.NewSource(
+                cli.Context(c),
+	    )
+            // Alternatively, just setup your config right here
+        }),
+    )
+    
+    // ... Load and use that source ...
+    conf := config.NewConfig()
+    conf.Load(clisrc)
+}
+```

config/source/cli/cli.go

@@ -0,0 +1,146 @@
+package cli
+
+import (
+	"flag"
+	"io/ioutil"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/imdario/mergo"
+	"github.com/micro/cli"
+	"github.com/micro/go-micro/cmd"
+	"github.com/micro/go-micro/config/source"
+)
+
+type cliSource struct {
+	opts source.Options
+	ctx  *cli.Context
+}
+
+func (c *cliSource) Read() (*source.ChangeSet, error) {
+	var changes map[string]interface{}
+
+	for _, name := range c.ctx.GlobalFlagNames() {
+		tmp := toEntry(name, c.ctx.GlobalGeneric(name))
+		mergo.Map(&changes, tmp) // need to sort error handling
+	}
+
+	for _, name := range c.ctx.FlagNames() {
+		tmp := toEntry(name, c.ctx.Generic(name))
+		mergo.Map(&changes, tmp) // need to sort error handling
+	}
+
+	b, err := c.opts.Encoder.Encode(changes)
+	if err != nil {
+		return nil, err
+	}
+
+	cs := &source.ChangeSet{
+		Format:    c.opts.Encoder.String(),
+		Data:      b,
+		Timestamp: time.Now(),
+		Source:    c.String(),
+	}
+	cs.Checksum = cs.Sum()
+
+	return cs, nil
+}
+
+func toEntry(name string, v interface{}) map[string]interface{} {
+	n := strings.ToLower(name)
+	keys := strings.FieldsFunc(n, split)
+	reverse(keys)
+	tmp := make(map[string]interface{})
+	for i, k := range keys {
+		if i == 0 {
+			tmp[k] = v
+			continue
+		}
+
+		tmp = map[string]interface{}{k: tmp}
+	}
+	return tmp
+}
+
+func reverse(ss []string) {
+	for i := len(ss)/2 - 1; i >= 0; i-- {
+		opp := len(ss) - 1 - i
+		ss[i], ss[opp] = ss[opp], ss[i]
+	}
+}
+
+func split(r rune) bool {
+	return r == '-' || r == '_'
+}
+
+func (c *cliSource) Watch() (source.Watcher, error) {
+	return source.NewNoopWatcher()
+}
+
+func (c *cliSource) String() string {
+	return "cli"
+}
+
+// NewSource returns a config source for integrating parsed flags from a micro/cli.Context.
+// Hyphens are delimiters for nesting, and all keys are lowercased. The assumption is that
+// command line flags have already been parsed.
+//
+// Example:
+//      cli.StringFlag{Name: "db-host"},
+//
+//
+//      {
+//          "database": {
+//              "host": "localhost"
+//          }
+//      }
+func NewSource(opts ...source.Option) source.Source {
+	options := source.NewOptions(opts...)
+
+	var ctx *cli.Context
+
+	c, ok := options.Context.Value(contextKey{}).(*cli.Context)
+	if ok {
+		ctx = c
+	}
+
+	// no context
+	if ctx == nil {
+		// get the default app/flags
+		app := cmd.App()
+		flags := app.Flags
+
+		// create flagset
+		set := flag.NewFlagSet(app.Name, flag.ContinueOnError)
+
+		// apply flags to set
+		for _, f := range flags {
+			f.Apply(set)
+		}
+
+		// parse flags
+		set.SetOutput(ioutil.Discard)
+		set.Parse(os.Args[1:])
+
+		// normalise flags
+		normalizeFlags(app.Flags, set)
+
+		// create context
+		ctx = cli.NewContext(app, set, nil)
+	}
+
+	return &cliSource{
+		ctx:  ctx,
+		opts: options,
+	}
+}
+
+// WithContext returns a new source with the context specified.
+// The assumption is that Context is retrieved within an app.Action function.
+func WithContext(ctx *cli.Context, opts ...source.Option) source.Source {
+	return &cliSource{
+		ctx:  ctx,
+		opts: source.NewOptions(opts...),
+	}
+}

config/source/cli/cli_test.go

@@ -0,0 +1,65 @@
+package cli
+
+import (
+	"encoding/json"
+	"os"
+	"testing"
+
+	"github.com/micro/cli"
+	"github.com/micro/go-micro/cmd"
+	"github.com/micro/go-micro/config/source"
+)
+
+func test(t *testing.T, withContext bool) {
+	var src source.Source
+
+	// setup app
+	app := cmd.App()
+	app.Name = "testapp"
+	app.Flags = []cli.Flag{
+		cli.StringFlag{Name: "db-host"},
+	}
+
+	// with context
+	if withContext {
+		// set action
+		app.Action = func(c *cli.Context) {
+			src = WithContext(c)
+		}
+
+		// run app
+		app.Run([]string{"run", "-db-host", "localhost"})
+		// no context
+	} else {
+		// set args
+		os.Args = []string{"run", "-db-host", "localhost"}
+		src = NewSource()
+	}
+
+	// test config
+	c, err := src.Read()
+	if err != nil {
+		t.Error(err)
+	}
+
+	var actual map[string]interface{}
+	if err := json.Unmarshal(c.Data, &actual); err != nil {
+		t.Error(err)
+	}
+
+	actualDB := actual["db"].(map[string]interface{})
+	if actualDB["host"] != "localhost" {
+		t.Errorf("expected localhost, got %v", actualDB["name"])
+	}
+
+}
+
+func TestCliSource(t *testing.T) {
+	// without context
+	test(t, false)
+}
+
+func TestCliSourceWithContext(t *testing.T) {
+	// with context
+	test(t, true)
+}

config/source/cli/options.go

@@ -0,0 +1,20 @@
+package cli
+
+import (
+	"context"
+
+	"github.com/micro/cli"
+	"github.com/micro/go-micro/config/source"
+)
+
+type contextKey struct{}
+
+// Context sets the cli context
+func Context(c *cli.Context) source.Option {
+	return func(o *source.Options) {
+		if o.Context == nil {
+			o.Context = context.Background()
+		}
+		o.Context = context.WithValue(o.Context, contextKey{}, c)
+	}
+}

config/source/cli/util.go

@@ -0,0 +1,50 @@
+package cli
+
+import (
+	"errors"
+	"flag"
+	"strings"
+
+	"github.com/micro/cli"
+)
+
+func copyFlag(name string, ff *flag.Flag, set *flag.FlagSet) {
+	switch ff.Value.(type) {
+	case *cli.StringSlice:
+	default:
+		set.Set(name, ff.Value.String())
+	}
+}
+
+func normalizeFlags(flags []cli.Flag, set *flag.FlagSet) error {
+	visited := make(map[string]bool)
+	set.Visit(func(f *flag.Flag) {
+		visited[f.Name] = true
+	})
+	for _, f := range flags {
+		parts := strings.Split(f.GetName(), ",")
+		if len(parts) == 1 {
+			continue
+		}
+		var ff *flag.Flag
+		for _, name := range parts {
+			name = strings.Trim(name, " ")
+			if visited[name] {
+				if ff != nil {
+					return errors.New("Cannot use two forms of the same flag: " + name + " " + ff.Name)
+				}
+				ff = set.Lookup(name)
+			}
+		}
+		if ff == nil {
+			continue
+		}
+		for _, name := range parts {
+			name = strings.Trim(name, " ")
+			if !visited[name] {
+				copyFlag(name, ff, set)
+			}
+		}
+	}
+	return nil
+}

config/source/consul/README.md

@@ -0,0 +1,49 @@
+# Consul Source
+
+The consul source reads config from consul key/values
+
+## Consul Format
+
+The consul source expects keys under the default prefix `/micro/config`
+
+Values are expected to be json
+
+```
+// set database
+consul kv put micro/config/database '{"address": "10.0.0.1", "port": 3306}'
+// set cache
+consul kv put micro/config/cache '{"address": "10.0.0.2", "port": 6379}'
+```
+
+Keys are split on `/` so access becomes
+
+```
+conf.Get("micro", "config", "database")
+```
+
+## New Source
+
+Specify source with data
+
+```go
+consulSource := consul.NewSource(
+	// optionally specify consul address; default to localhost:8500
+	consul.WithAddress("10.0.0.10:8500"),
+	// optionally specify prefix; defaults to /micro/config
+	consul.WithPrefix("/my/prefix"),
+  // optionally strip the provided prefix from the keys, defaults to false
+  consul.StripPrefix(true),
+)
+```
+
+## Load Source
+
+Load the source into config
+
+```go
+// Create new config
+conf := config.NewConfig()
+
+// Load file source
+conf.Load(consulSource)
+```

config/source/consul/consul.go

@@ -0,0 +1,121 @@
+package consul
+
+import (
+	"fmt"
+	"net"
+	"time"
+
+	"github.com/hashicorp/consul/api"
+	"github.com/micro/go-micro/config/source"
+)
+
+// Currently a single consul reader
+type consul struct {
+	prefix      string
+	stripPrefix string
+	addr        string
+	opts        source.Options
+	client      *api.Client
+}
+
+var (
+	// DefaultPrefix is the prefix that consul keys will be assumed to have if you
+	// haven't specified one
+	DefaultPrefix = "/micro/config/"
+)
+
+func (c *consul) Read() (*source.ChangeSet, error) {
+	kv, _, err := c.client.KV().List(c.prefix, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	if kv == nil || len(kv) == 0 {
+		return nil, fmt.Errorf("source not found: %s", c.prefix)
+	}
+
+	data, err := makeMap(c.opts.Encoder, kv, c.stripPrefix)
+	if err != nil {
+		return nil, fmt.Errorf("error reading data: %v", err)
+	}
+
+	b, err := c.opts.Encoder.Encode(data)
+	if err != nil {
+		return nil, fmt.Errorf("error reading source: %v", err)
+	}
+
+	cs := &source.ChangeSet{
+		Timestamp: time.Now(),
+		Format:    c.opts.Encoder.String(),
+		Source:    c.String(),
+		Data:      b,
+	}
+	cs.Checksum = cs.Sum()
+
+	return cs, nil
+}
+
+func (c *consul) String() string {
+	return "consul"
+}
+
+func (c *consul) Watch() (source.Watcher, error) {
+	w, err := newWatcher(c.prefix, c.addr, c.String(), c.stripPrefix, c.opts.Encoder)
+	if err != nil {
+		return nil, err
+	}
+	return w, nil
+}
+
+// NewSource creates a new consul source
+func NewSource(opts ...source.Option) source.Source {
+	options := source.NewOptions(opts...)
+
+	// use default config
+	config := api.DefaultConfig()
+
+	// check if there are any addrs
+	a, ok := options.Context.Value(addressKey{}).(string)
+	if ok {
+		addr, port, err := net.SplitHostPort(a)
+		if ae, ok := err.(*net.AddrError); ok && ae.Err == "missing port in address" {
+			port = "8500"
+			addr = a
+			config.Address = fmt.Sprintf("%s:%s", addr, port)
+		} else if err == nil {
+			config.Address = fmt.Sprintf("%s:%s", addr, port)
+		}
+	}
+
+	dc, ok := options.Context.Value(dcKey{}).(string)
+	if ok {
+		config.Datacenter = dc
+	}
+
+	token, ok := options.Context.Value(tokenKey{}).(string)
+	if ok {
+		config.Token = token
+	}
+
+	// create the client
+	client, _ := api.NewClient(config)
+
+	prefix := DefaultPrefix
+	sp := ""
+	f, ok := options.Context.Value(prefixKey{}).(string)
+	if ok {
+		prefix = f
+	}
+
+	if b, ok := options.Context.Value(stripPrefixKey{}).(bool); ok && b {
+		sp = prefix
+	}
+
+	return &consul{
+		prefix:      prefix,
+		stripPrefix: sp,
+		addr:        config.Address,
+		opts:        options,
+		client:      client,
+	}
+}

config/source/consul/options.go

@@ -0,0 +1,63 @@
+package consul
+
+import (
+	"context"
+
+	"github.com/micro/go-micro/config/source"
+)
+
+type addressKey struct{}
+type prefixKey struct{}
+type stripPrefixKey struct{}
+type dcKey struct{}
+type tokenKey struct{}
+
+// WithAddress sets the consul address
+func WithAddress(a string) source.Option {
+	return func(o *source.Options) {
+		if o.Context == nil {
+			o.Context = context.Background()
+		}
+		o.Context = context.WithValue(o.Context, addressKey{}, a)
+	}
+}
+
+// WithPrefix sets the key prefix to use
+func WithPrefix(p string) source.Option {
+	return func(o *source.Options) {
+		if o.Context == nil {
+			o.Context = context.Background()
+		}
+		o.Context = context.WithValue(o.Context, prefixKey{}, p)
+	}
+}
+
+// StripPrefix indicates whether to remove the prefix from config entries, or leave it in place.
+func StripPrefix(strip bool) source.Option {
+	return func(o *source.Options) {
+		if o.Context == nil {
+			o.Context = context.Background()
+		}
+
+		o.Context = context.WithValue(o.Context, stripPrefixKey{}, strip)
+	}
+}
+
+func WithDatacenter(p string) source.Option {
+	return func(o *source.Options) {
+		if o.Context == nil {
+			o.Context = context.Background()
+		}
+		o.Context = context.WithValue(o.Context, dcKey{}, p)
+	}
+}
+
+// WithToken sets the key token to use
+func WithToken(p string) source.Option {
+	return func(o *source.Options) {
+		if o.Context == nil {
+			o.Context = context.Background()
+		}
+		o.Context = context.WithValue(o.Context, tokenKey{}, p)
+	}
+}

config/source/consul/util.go

@@ -0,0 +1,49 @@
+package consul
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/hashicorp/consul/api"
+	"github.com/micro/go-micro/config/encoder"
+)
+
+func makeMap(e encoder.Encoder, kv api.KVPairs, stripPrefix string) (map[string]interface{}, error) {
+	data := make(map[string]interface{})
+
+	// consul guarantees lexicographic order, so no need to sort
+	for _, v := range kv {
+		pathString := strings.TrimPrefix(strings.TrimPrefix(v.Key, stripPrefix), "/")
+		var val map[string]interface{}
+
+		// ensure a valid value is stored at this location
+		if len(v.Value) > 0 {
+			if err := e.Decode(v.Value, &val); err != nil {
+				return nil, fmt.Errorf("faild decode value. path: %s, error: %s", pathString, err)
+			}
+		}
+
+		// set target at the root
+		target := data
+
+		// then descend to the target location, creating as we go, if need be
+		if pathString != "" {
+			path := strings.Split(pathString, "/")
+			// find (or create) the location we want to put this value at
+			for _, dir := range path {
+				if _, ok := target[dir]; !ok {
+					target[dir] = make(map[string]interface{})
+				}
+				target = target[dir].(map[string]interface{})
+			}
+
+		}
+
+		// copy over the keys from the value
+		for k := range val {
+			target[k] = val[k]
+		}
+	}
+
+	return data, nil
+}

config/source/consul/watcher.go

@@ -0,0 +1,96 @@
+package consul
+
+import (
+	"errors"
+	"time"
+
+	"github.com/hashicorp/consul/api"
+	"github.com/hashicorp/consul/api/watch"
+	"github.com/micro/go-micro/config/encoder"
+	"github.com/micro/go-micro/config/source"
+)
+
+type watcher struct {
+	e           encoder.Encoder
+	name        string
+	stripPrefix string
+
+	wp   *watch.Plan
+	ch   chan *source.ChangeSet
+	exit chan bool
+}
+
+func newWatcher(key, addr, name, stripPrefix string, e encoder.Encoder) (source.Watcher, error) {
+	w := &watcher{
+		e:           e,
+		name:        name,
+		stripPrefix: stripPrefix,
+		ch:          make(chan *source.ChangeSet),
+		exit:        make(chan bool),
+	}
+
+	wp, err := watch.Parse(map[string]interface{}{"type": "keyprefix", "prefix": key})
+	if err != nil {
+		return nil, err
+	}
+
+	wp.Handler = w.handle
+
+	// wp.Run is a blocking call and will prevent newWatcher from returning
+	go wp.Run(addr)
+
+	w.wp = wp
+
+	return w, nil
+}
+
+func (w *watcher) handle(idx uint64, data interface{}) {
+	if data == nil {
+		return
+	}
+
+	kvs, ok := data.(api.KVPairs)
+	if !ok {
+		return
+	}
+
+	d, err := makeMap(w.e, kvs, w.stripPrefix)
+	if err != nil {
+		return
+	}
+
+	b, err := w.e.Encode(d)
+	if err != nil {
+		return
+	}
+
+	cs := &source.ChangeSet{
+		Timestamp: time.Now(),
+		Format:    w.e.String(),
+		Source:    w.name,
+		Data:      b,
+	}
+	cs.Checksum = cs.Sum()
+
+	w.ch <- cs
+}
+
+func (w *watcher) Next() (*source.ChangeSet, error) {
+	select {
+	case cs := <-w.ch:
+		return cs, nil
+	case <-w.exit:
+		return nil, errors.New("watcher stopped")
+	}
+}
+
+func (w *watcher) Stop() error {
+	select {
+	case <-w.exit:
+		return nil
+	default:
+		w.wp.Stop()
+		close(w.exit)
+	}
+	return nil
+}

config/source/env/README.md

@@ -0,0 +1,96 @@
+# Env Source
+
+The env source reads config from environment variables
+
+## Format
+
+We expect environment variables to be in the standard format of FOO=bar
+
+Keys are converted to lowercase and split on underscore.
+
+
+### Example
+
+```
+DATABASE_ADDRESS=127.0.0.1
+DATABASE_PORT=3306
+```
+
+Becomes
+
+```json
+{
+    "database": {
+        "address": "127.0.0.1",
+        "port": 3306
+    }
+}
+```
+
+## Prefixes
+
+Environment variables can be namespaced so we only have access to a subset. Two options are available:
+
+```
+WithPrefix(p ...string)
+WithStrippedPrefix(p ...string)
+```
+
+The former will preserve the prefix and make it a top level key in the config. The latter eliminates the prefix, reducing the nesting by one. 
+
+#### Example:
+
+Given ENVs of:
+
+```
+APP_DATABASE_ADDRESS=127.0.0.1
+APP_DATABASE_PORT=3306
+VAULT_ADDR=vault:1337
+```
+
+and a source initialized as follows:
+
+```
+src := env.NewSource(
+    env.WithPrefix("VAULT"),
+    env.WithStrippedPrefix("APP"),
+)
+```
+
+The resulting config will be:
+
+```
+{
+    "database": {
+        "address": "127.0.0.1",
+        "port": 3306
+    },
+    "vault": {
+        "addr": "vault:1337"
+    }
+}
+```
+
+
+## New Source
+
+Specify source with data
+
+```go
+src := env.NewSource(
+	// optionally specify prefix
+	env.WithPrefix("MICRO"),
+)
+```
+
+## Load Source
+
+Load the source into config
+
+```go
+// Create new config
+conf := config.NewConfig()
+
+// Load file source
+conf.Load(src)
+```

config/source/env/env.go

@@ -0,0 +1,142 @@
+package env
+
+import (
+	"os"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/imdario/mergo"
+	"github.com/micro/go-micro/config/source"
+)
+
+var (
+	DefaultPrefixes = []string{}
+)
+
+type env struct {
+	prefixes         []string
+	strippedPrefixes []string
+	opts             source.Options
+}
+
+func (e *env) Read() (*source.ChangeSet, error) {
+	var changes map[string]interface{}
+
+	for _, env := range os.Environ() {
+
+		if len(e.prefixes) > 0 || len(e.strippedPrefixes) > 0 {
+			notFound := true
+
+			if _, ok := matchPrefix(e.prefixes, env); ok {
+				notFound = false
+			}
+
+			if match, ok := matchPrefix(e.strippedPrefixes, env); ok {
+				env = strings.TrimPrefix(env, match)
+				notFound = false
+			}
+
+			if notFound {
+				continue
+			}
+		}
+
+		pair := strings.SplitN(env, "=", 2)
+		value := pair[1]
+		keys := strings.Split(strings.ToLower(pair[0]), "_")
+		reverse(keys)
+
+		tmp := make(map[string]interface{})
+		for i, k := range keys {
+			if i == 0 {
+				if intValue, err := strconv.Atoi(value); err == nil {
+					tmp[k] = intValue
+				} else if boolValue, err := strconv.ParseBool(value); err == nil {
+					tmp[k] = boolValue
+				} else {
+					tmp[k] = value
+				}
+				continue
+			}
+
+			tmp = map[string]interface{}{k: tmp}
+		}
+
+		if err := mergo.Map(&changes, tmp); err != nil {
+			return nil, err
+		}
+	}
+
+	b, err := e.opts.Encoder.Encode(changes)
+	if err != nil {
+		return nil, err
+	}
+
+	cs := &source.ChangeSet{
+		Format:    e.opts.Encoder.String(),
+		Data:      b,
+		Timestamp: time.Now(),
+		Source:    e.String(),
+	}
+	cs.Checksum = cs.Sum()
+
+	return cs, nil
+}
+
+func matchPrefix(pre []string, s string) (string, bool) {
+	for _, p := range pre {
+		if strings.HasPrefix(s, p) {
+			return p, true
+		}
+	}
+
+	return "", false
+}
+
+func reverse(ss []string) {
+	for i := len(ss)/2 - 1; i >= 0; i-- {
+		opp := len(ss) - 1 - i
+		ss[i], ss[opp] = ss[opp], ss[i]
+	}
+}
+
+func (e *env) Watch() (source.Watcher, error) {
+	return newWatcher()
+}
+
+func (e *env) String() string {
+	return "env"
+}
+
+// NewSource returns a config source for parsing ENV variables.
+// Underscores are delimiters for nesting, and all keys are lowercased.
+//
+// Example:
+//      "DATABASE_SERVER_HOST=localhost" will convert to
+//
+//      {
+//          "database": {
+//              "server": {
+//                  "host": "localhost"
+//              }
+//          }
+//      }
+func NewSource(opts ...source.Option) source.Source {
+	options := source.NewOptions(opts...)
+
+	var sp []string
+	var pre []string
+	if p, ok := options.Context.Value(strippedPrefixKey{}).([]string); ok {
+		sp = p
+	}
+
+	if p, ok := options.Context.Value(prefixKey{}).([]string); ok {
+		pre = p
+	}
+
+	if len(sp) > 0 || len(pre) > 0 {
+		pre = append(pre, DefaultPrefixes...)
+	}
+	return &env{prefixes: pre, strippedPrefixes: sp, opts: options}
+}

config/source/env/env_test.go

@@ -0,0 +1,112 @@
+package env
+
+import (
+	"encoding/json"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/micro/go-micro/config/source"
+)
+
+func TestEnv_Read(t *testing.T) {
+	expected := map[string]map[string]string{
+		"database": {
+			"host":       "localhost",
+			"password":   "password",
+			"datasource": "user:password@tcp(localhost:port)/db?charset=utf8mb4&parseTime=True&loc=Local",
+		},
+	}
+
+	os.Setenv("DATABASE_HOST", "localhost")
+	os.Setenv("DATABASE_PASSWORD", "password")
+	os.Setenv("DATABASE_DATASOURCE", "user:password@tcp(localhost:port)/db?charset=utf8mb4&parseTime=True&loc=Local")
+
+	source := NewSource()
+	c, err := source.Read()
+	if err != nil {
+		t.Error(err)
+	}
+
+	var actual map[string]interface{}
+	if err := json.Unmarshal(c.Data, &actual); err != nil {
+		t.Error(err)
+	}
+
+	actualDB := actual["database"].(map[string]interface{})
+
+	for k, v := range expected["database"] {
+		a := actualDB[k]
+
+		if a != v {
+			t.Errorf("expected %v got %v", v, a)
+		}
+	}
+}
+
+func TestEnvvar_Prefixes(t *testing.T) {
+	os.Setenv("APP_DATABASE_HOST", "localhost")
+	os.Setenv("APP_DATABASE_PASSWORD", "password")
+	os.Setenv("VAULT_ADDR", "vault:1337")
+	os.Setenv("MICRO_REGISTRY", "mdns")
+
+	var prefixtests = []struct {
+		prefixOpts   []source.Option
+		expectedKeys []string
+	}{
+		{[]source.Option{WithPrefix("APP", "MICRO")}, []string{"app", "micro"}},
+		{[]source.Option{WithPrefix("MICRO"), WithStrippedPrefix("APP")}, []string{"database", "micro"}},
+		{[]source.Option{WithPrefix("MICRO"), WithStrippedPrefix("APP")}, []string{"database", "micro"}},
+	}
+
+	for _, pt := range prefixtests {
+		source := NewSource(pt.prefixOpts...)
+
+		c, err := source.Read()
+		if err != nil {
+			t.Error(err)
+		}
+
+		var actual map[string]interface{}
+		if err := json.Unmarshal(c.Data, &actual); err != nil {
+			t.Error(err)
+		}
+
+		// assert other prefixes ignored
+		if l := len(actual); l != len(pt.expectedKeys) {
+			t.Errorf("expected %v top keys, got %v", len(pt.expectedKeys), l)
+		}
+
+		for _, k := range pt.expectedKeys {
+			if !containsKey(actual, k) {
+				t.Errorf("expected key %v, not found", k)
+			}
+		}
+	}
+}
+
+func TestEnvvar_WatchNextNoOpsUntilStop(t *testing.T) {
+	source := NewSource(WithStrippedPrefix("GOMICRO_"))
+	w, err := source.Watch()
+	if err != nil {
+		t.Error(err)
+	}
+
+	go func() {
+		time.Sleep(50 * time.Millisecond)
+		w.Stop()
+	}()
+
+	if _, err := w.Next(); err.Error() != "watcher stopped" {
+		t.Errorf("expected watcher stopped error, got %v", err)
+	}
+}
+
+func containsKey(m map[string]interface{}, s string) bool {
+	for k := range m {
+		if k == s {
+			return true
+		}
+	}
+	return false
+}

config/source/env/options.go

@@ -0,0 +1,49 @@
+package env
+
+import (
+	"context"
+
+	"strings"
+
+	"github.com/micro/go-micro/config/source"
+)
+
+type strippedPrefixKey struct{}
+type prefixKey struct{}
+
+// WithStrippedPrefix sets the environment variable prefixes to scope to.
+// These prefixes will be removed from the actual config entries.
+func WithStrippedPrefix(p ...string) source.Option {
+	return func(o *source.Options) {
+		if o.Context == nil {
+			o.Context = context.Background()
+		}
+
+		o.Context = context.WithValue(o.Context, strippedPrefixKey{}, appendUnderscore(p))
+	}
+}
+
+// WithPrefix sets the environment variable prefixes to scope to.
+// These prefixes will not be removed. Each prefix will be considered a top level config entry.
+func WithPrefix(p ...string) source.Option {
+	return func(o *source.Options) {
+		if o.Context == nil {
+			o.Context = context.Background()
+		}
+		o.Context = context.WithValue(o.Context, prefixKey{}, appendUnderscore(p))
+	}
+}
+
+func appendUnderscore(prefixes []string) []string {
+	var result []string
+	for _, p := range prefixes {
+		if !strings.HasSuffix(p, "_") {
+			result = append(result, p+"_")
+			continue
+		}
+
+		result = append(result, p)
+	}
+
+	return result
+}

config/source/env/watcher.go

@@ -0,0 +1,26 @@
+package env
+
+import (
+	"errors"
+
+	"github.com/micro/go-micro/config/source"
+)
+
+type watcher struct {
+	exit chan struct{}
+}
+
+func (w *watcher) Next() (*source.ChangeSet, error) {
+	<-w.exit
+
+	return nil, errors.New("watcher stopped")
+}
+
+func (w *watcher) Stop() error {
+	close(w.exit)
+	return nil
+}
+
+func newWatcher() (source.Watcher, error) {
+	return &watcher{exit: make(chan struct{})}, nil
+}

config/source/etcd/README.md

@@ -0,0 +1,51 @@
+# Etcd Source
+
+The etcd source reads config from etcd key/values
+
+This source supports etcd version 3 and beyond.
+
+## Etcd Format
+
+The etcd source expects keys under the default prefix `/micro/config` (prefix can be changed)
+
+Values are expected to be JSON
+
+```
+// set database
+etcdctl put /micro/config/database '{"address": "10.0.0.1", "port": 3306}'
+// set cache
+etcdctl put /micro/config/cache '{"address": "10.0.0.2", "port": 6379}'
+```
+
+Keys are split on `/` so access becomes
+
+```
+conf.Get("micro", "config", "database")
+```
+
+## New Source
+
+Specify source with data
+
+```go
+etcdSource := etcd.NewSource(
+	// optionally specify etcd address; default to localhost:8500
+	etcd.WithAddress("10.0.0.10:8500"),
+	// optionally specify prefix; defaults to /micro/config
+	etcd.WithPrefix("/my/prefix"),
+	// optionally strip the provided prefix from the keys, defaults to false
+	etcd.StripPrefix(true),
+)
+```
+
+## Load Source
+
+Load the source into config
+
+```go
+// Create new config
+conf := config.NewConfig()
+
+// Load file source
+conf.Load(etcdSource)
+```

config/source/etcd/etcd.go

@@ -0,0 +1,134 @@
+package etcd
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"time"
+
+	"github.com/micro/go-micro/config/source"
+	cetcd "go.etcd.io/etcd/clientv3"
+	"go.etcd.io/etcd/mvcc/mvccpb"
+)
+
+// Currently a single etcd reader
+type etcd struct {
+	prefix      string
+	stripPrefix string
+	opts        source.Options
+	client      *cetcd.Client
+	cerr        error
+}
+
+var (
+	DefaultPrefix = "/micro/config/"
+)
+
+func (c *etcd) Read() (*source.ChangeSet, error) {
+	if c.cerr != nil {
+		return nil, c.cerr
+	}
+
+	rsp, err := c.client.Get(context.Background(), c.prefix, cetcd.WithPrefix())
+	if err != nil {
+		return nil, err
+	}
+
+	if rsp == nil || len(rsp.Kvs) == 0 {
+		return nil, fmt.Errorf("source not found: %s", c.prefix)
+	}
+
+	var kvs []*mvccpb.KeyValue
+	for _, v := range rsp.Kvs {
+		kvs = append(kvs, (*mvccpb.KeyValue)(v))
+	}
+
+	data := makeMap(c.opts.Encoder, kvs, c.stripPrefix)
+
+	b, err := c.opts.Encoder.Encode(data)
+	if err != nil {
+		return nil, fmt.Errorf("error reading source: %v", err)
+	}
+
+	cs := &source.ChangeSet{
+		Timestamp: time.Now(),
+		Source:    c.String(),
+		Data:      b,
+		Format:    c.opts.Encoder.String(),
+	}
+	cs.Checksum = cs.Sum()
+
+	return cs, nil
+}
+
+func (c *etcd) String() string {
+	return "etcd"
+}
+
+func (c *etcd) Watch() (source.Watcher, error) {
+	if c.cerr != nil {
+		return nil, c.cerr
+	}
+	cs, err := c.Read()
+	if err != nil {
+		return nil, err
+	}
+	return newWatcher(c.prefix, c.stripPrefix, c.client.Watcher, cs, c.opts)
+}
+
+func NewSource(opts ...source.Option) source.Source {
+	options := source.NewOptions(opts...)
+
+	var endpoints []string
+
+	// check if there are any addrs
+	addrs, ok := options.Context.Value(addressKey{}).([]string)
+	if ok {
+		for _, a := range addrs {
+			addr, port, err := net.SplitHostPort(a)
+			if ae, ok := err.(*net.AddrError); ok && ae.Err == "missing port in address" {
+				port = "2379"
+				addr = a
+				endpoints = append(endpoints, fmt.Sprintf("%s:%s", addr, port))
+			} else if err == nil {
+				endpoints = append(endpoints, fmt.Sprintf("%s:%s", addr, port))
+			}
+		}
+	}
+
+	if len(endpoints) == 0 {
+		endpoints = []string{"localhost:2379"}
+	}
+
+	config := cetcd.Config{
+		Endpoints: endpoints,
+	}
+
+	u, ok := options.Context.Value(authKey{}).(*authCreds)
+	if ok {
+		config.Username = u.Username
+		config.Password = u.Password
+	}
+
+	// use default config
+	client, err := cetcd.New(config)
+
+	prefix := DefaultPrefix
+	sp := ""
+	f, ok := options.Context.Value(prefixKey{}).(string)
+	if ok {
+		prefix = f
+	}
+
+	if b, ok := options.Context.Value(stripPrefixKey{}).(bool); ok && b {
+		sp = prefix
+	}
+
+	return &etcd{
+		prefix:      prefix,
+		stripPrefix: sp,
+		opts:        options,
+		client:      client,
+		cerr:        err,
+	}
+}

config/source/etcd/options.go

@@ -0,0 +1,58 @@
+package etcd
+
+import (
+	"context"
+
+	"github.com/micro/go-micro/config/source"
+)
+
+type addressKey struct{}
+type prefixKey struct{}
+type stripPrefixKey struct{}
+type authKey struct{}
+
+type authCreds struct {
+	Username string
+	Password string
+}
+
+// WithAddress sets the consul address
+func WithAddress(a ...string) source.Option {
+	return func(o *source.Options) {
+		if o.Context == nil {
+			o.Context = context.Background()
+		}
+		o.Context = context.WithValue(o.Context, addressKey{}, a)
+	}
+}
+
+// WithPrefix sets the key prefix to use
+func WithPrefix(p string) source.Option {
+	return func(o *source.Options) {
+		if o.Context == nil {
+			o.Context = context.Background()
+		}
+		o.Context = context.WithValue(o.Context, prefixKey{}, p)
+	}
+}
+
+// StripPrefix indicates whether to remove the prefix from config entries, or leave it in place.
+func StripPrefix(strip bool) source.Option {
+	return func(o *source.Options) {
+		if o.Context == nil {
+			o.Context = context.Background()
+		}
+
+		o.Context = context.WithValue(o.Context, stripPrefixKey{}, strip)
+	}
+}
+
+// Auth allows you to specify username/password
+func Auth(username, password string) source.Option {
+	return func(o *source.Options) {
+		if o.Context == nil {
+			o.Context = context.Background()
+		}
+		o.Context = context.WithValue(o.Context, authKey{}, &authCreds{Username: username, Password: password})
+	}
+}

config/source/etcd/util.go

@@ -0,0 +1,89 @@
+package etcd
+
+import (
+	"strings"
+
+	"github.com/micro/go-micro/config/encoder"
+	"go.etcd.io/etcd/clientv3"
+	"go.etcd.io/etcd/mvcc/mvccpb"
+)
+
+func makeEvMap(e encoder.Encoder, data map[string]interface{}, kv []*clientv3.Event, stripPrefix string) map[string]interface{} {
+	if data == nil {
+		data = make(map[string]interface{})
+	}
+
+	for _, v := range kv {
+		switch mvccpb.Event_EventType(v.Type) {
+		case mvccpb.DELETE:
+			data = update(e, data, (*mvccpb.KeyValue)(v.Kv), "delete", stripPrefix)
+		default:
+			data = update(e, data, (*mvccpb.KeyValue)(v.Kv), "insert", stripPrefix)
+		}
+	}
+
+	return data
+}
+
+func makeMap(e encoder.Encoder, kv []*mvccpb.KeyValue, stripPrefix string) map[string]interface{} {
+	data := make(map[string]interface{})
+
+	for _, v := range kv {
+		data = update(e, data, v, "put", stripPrefix)
+	}
+
+	return data
+}
+
+func update(e encoder.Encoder, data map[string]interface{}, v *mvccpb.KeyValue, action, stripPrefix string) map[string]interface{} {
+	// remove prefix if non empty, and ensure leading / is removed as well
+	vkey := strings.TrimPrefix(strings.TrimPrefix(string(v.Key), stripPrefix), "/")
+	// split on prefix
+	haveSplit := strings.Contains(vkey, "/")
+	keys := strings.Split(vkey, "/")
+
+	var vals interface{}
+	e.Decode(v.Value, &vals)
+
+	if !haveSplit && len(keys) == 1 {
+		switch action {
+		case "delete":
+			data = make(map[string]interface{})
+		default:
+			v, ok := vals.(map[string]interface{})
+			if ok {
+				data = v
+			}
+		}
+		return data
+	}
+
+	// set data for first iteration
+	kvals := data
+	// iterate the keys and make maps
+	for i, k := range keys {
+		kval, ok := kvals[k].(map[string]interface{})
+		if !ok {
+			// create next map
+			kval = make(map[string]interface{})
+			// set it
+			kvals[k] = kval
+		}
+
+		// last key: write vals
+		if l := len(keys) - 1; i == l {
+			switch action {
+			case "delete":
+				delete(kvals, k)
+			default:
+				kvals[k] = vals
+			}
+			break
+		}
+
+		// set kvals for next iterator
+		kvals = kval
+	}
+
+	return data
+}

config/source/etcd/watcher.go

@@ -0,0 +1,113 @@
+package etcd
+
+import (
+	"context"
+	"errors"
+	"sync"
+	"time"
+
+	"github.com/micro/go-micro/config/source"
+	cetcd "go.etcd.io/etcd/clientv3"
+)
+
+type watcher struct {
+	opts        source.Options
+	name        string
+	stripPrefix string
+
+	sync.RWMutex
+	cs *source.ChangeSet
+
+	ch   chan *source.ChangeSet
+	exit chan bool
+}
+
+func newWatcher(key, strip string, wc cetcd.Watcher, cs *source.ChangeSet, opts source.Options) (source.Watcher, error) {
+	w := &watcher{
+		opts:        opts,
+		name:        "etcd",
+		stripPrefix: strip,
+		cs:          cs,
+		ch:          make(chan *source.ChangeSet),
+		exit:        make(chan bool),
+	}
+
+	ch := wc.Watch(context.Background(), key, cetcd.WithPrefix())
+
+	go w.run(wc, ch)
+
+	return w, nil
+}
+
+func (w *watcher) handle(evs []*cetcd.Event) {
+	w.RLock()
+	data := w.cs.Data
+	w.RUnlock()
+
+	var vals map[string]interface{}
+
+	// unpackage existing changeset
+	if err := w.opts.Encoder.Decode(data, &vals); err != nil {
+		return
+	}
+
+	// update base changeset
+	d := makeEvMap(w.opts.Encoder, vals, evs, w.stripPrefix)
+
+	// pack the changeset
+	b, err := w.opts.Encoder.Encode(d)
+	if err != nil {
+		return
+	}
+
+	// create new changeset
+	cs := &source.ChangeSet{
+		Timestamp: time.Now(),
+		Source:    w.name,
+		Data:      b,
+		Format:    w.opts.Encoder.String(),
+	}
+	cs.Checksum = cs.Sum()
+
+	// set base change set
+	w.Lock()
+	w.cs = cs
+	w.Unlock()
+
+	// send update
+	w.ch <- cs
+}
+
+func (w *watcher) run(wc cetcd.Watcher, ch cetcd.WatchChan) {
+	for {
+		select {
+		case rsp, ok := <-ch:
+			if !ok {
+				return
+			}
+			w.handle(rsp.Events)
+		case <-w.exit:
+			wc.Close()
+			return
+		}
+	}
+}
+
+func (w *watcher) Next() (*source.ChangeSet, error) {
+	select {
+	case cs := <-w.ch:
+		return cs, nil
+	case <-w.exit:
+		return nil, errors.New("watcher stopped")
+	}
+}
+
+func (w *watcher) Stop() error {
+	select {
+	case <-w.exit:
+		return nil
+	default:
+		close(w.exit)
+	}
+	return nil
+}

config/source/file/README.md

@@ -0,0 +1,70 @@
+# File Source
+
+The file source reads config from a file. 
+
+It uses the File extension to determine the Format e.g `config.yaml` has the yaml format. 
+It does not make use of encoders or interpet the file data. If a file extension is not present 
+the source Format will default to the Encoder in options.
+
+## Example
+
+A config file format in json
+
+```json
+{
+    "hosts": {
+        "database": {
+            "address": "10.0.0.1",
+            "port": 3306
+        },
+        "cache": {
+            "address": "10.0.0.2",
+            "port": 6379
+        }
+    }
+}
+```
+
+## New Source
+
+Specify file source with path to file. Path is optional and will default to `config.json`
+
+```go
+fileSource := file.NewSource(
+	file.WithPath("/tmp/config.json"),
+)
+```
+
+## File Format
+
+To load different file formats e.g yaml, toml, xml simply specify them with their extension
+
+```
+fileSource := file.NewSource(
+        file.WithPath("/tmp/config.yaml"),
+)
+```
+
+If you want to specify a file without extension, ensure you set the encoder to the same format
+
+```
+e := toml.NewEncoder()
+
+fileSource := file.NewSource(
+        file.WithPath("/tmp/config"),
+	source.WithEncoder(e),
+)
+```
+
+## Load Source
+
+Load the source into config
+
+```go
+// Create new config
+conf := config.NewConfig()
+
+// Load file source
+conf.Load(fileSource)
+```
+

config/source/file/file.go

@@ -0,0 +1,65 @@
+// Package file is a file source. Expected format is json
+package file
+
+import (
+	"io/ioutil"
+	"os"
+
+	"github.com/micro/go-micro/config/source"
+)
+
+type file struct {
+	path string
+	opts source.Options
+}
+
+var (
+	DefaultPath = "config.json"
+)
+
+func (f *file) Read() (*source.ChangeSet, error) {
+	fh, err := os.Open(f.path)
+	if err != nil {
+		return nil, err
+	}
+	defer fh.Close()
+	b, err := ioutil.ReadAll(fh)
+	if err != nil {
+		return nil, err
+	}
+	info, err := fh.Stat()
+	if err != nil {
+		return nil, err
+	}
+
+	cs := &source.ChangeSet{
+		Format:    format(f.path, f.opts.Encoder),
+		Source:    f.String(),
+		Timestamp: info.ModTime(),
+		Data:      b,
+	}
+	cs.Checksum = cs.Sum()
+
+	return cs, nil
+}
+
+func (f *file) String() string {
+	return "file"
+}
+
+func (f *file) Watch() (source.Watcher, error) {
+	if _, err := os.Stat(f.path); err != nil {
+		return nil, err
+	}
+	return newWatcher(f)
+}
+
+func NewSource(opts ...source.Option) source.Source {
+	options := source.NewOptions(opts...)
+	path := DefaultPath
+	f, ok := options.Context.Value(filePathKey{}).(string)
+	if ok {
+		path = f
+	}
+	return &file{opts: options, path: path}
+}

config/source/file/file_test.go

@@ -0,0 +1,37 @@
+package file
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+)
+
+func TestFile(t *testing.T) {
+	data := []byte(`{"foo": "bar"}`)
+	path := filepath.Join(os.TempDir(), fmt.Sprintf("file.%d", time.Now().UnixNano()))
+	fh, err := os.Create(path)
+	if err != nil {
+		t.Error(err)
+	}
+	defer func() {
+		fh.Close()
+		os.Remove(path)
+	}()
+
+	_, err = fh.Write(data)
+	if err != nil {
+		t.Error(err)
+	}
+
+	f := NewSource(WithPath(path))
+	c, err := f.Read()
+	if err != nil {
+		t.Error(err)
+	}
+	t.Logf("%+v", c)
+	if string(c.Data) != string(data) {
+		t.Error("data from file does not match")
+	}
+}

config/source/file/format.go

@@ -0,0 +1,15 @@
+package file
+
+import (
+	"strings"
+
+	"github.com/micro/go-micro/config/encoder"
+)
+
+func format(p string, e encoder.Encoder) string {
+	parts := strings.Split(p, ".")
+	if len(parts) > 1 {
+		return parts[len(parts)-1]
+	}
+	return e.String()
+}

config/source/file/format_test.go

@@ -0,0 +1,31 @@
+package file
+
+import (
+	"testing"
+
+	"github.com/micro/go-micro/config/source"
+)
+
+func TestFormat(t *testing.T) {
+	opts := source.NewOptions()
+	e := opts.Encoder
+
+	testCases := []struct {
+		p string
+		f string
+	}{
+		{"/foo/bar.json", "json"},
+		{"/foo/bar.yaml", "yaml"},
+		{"/foo/bar.xml", "xml"},
+		{"/foo/bar.conf.ini", "ini"},
+		{"conf", e.String()},
+	}
+
+	for _, d := range testCases {
+		f := format(d.p, e)
+		if f != d.f {
+			t.Fatalf("%s: expected %s got %s", d.p, d.f, f)
+		}
+	}
+
+}

config/source/file/options.go

@@ -0,0 +1,19 @@
+package file
+
+import (
+	"context"
+
+	"github.com/micro/go-micro/config/source"
+)
+
+type filePathKey struct{}
+
+// WithPath sets the path to file
+func WithPath(p string) source.Option {
+	return func(o *source.Options) {
+		if o.Context == nil {
+			o.Context = context.Background()
+		}
+		o.Context = context.WithValue(o.Context, filePathKey{}, p)
+	}
+}

config/source/file/watcher.go

@@ -0,0 +1,66 @@
+package file
+
+import (
+	"errors"
+	"os"
+
+	"github.com/fsnotify/fsnotify"
+	"github.com/micro/go-micro/config/source"
+)
+
+type watcher struct {
+	f *file
+
+	fw   *fsnotify.Watcher
+	exit chan bool
+}
+
+func newWatcher(f *file) (source.Watcher, error) {
+	fw, err := fsnotify.NewWatcher()
+	if err != nil {
+		return nil, err
+	}
+
+	fw.Add(f.path)
+
+	return &watcher{
+		f:    f,
+		fw:   fw,
+		exit: make(chan bool),
+	}, nil
+}
+
+func (w *watcher) Next() (*source.ChangeSet, error) {
+	// is it closed?
+	select {
+	case <-w.exit:
+		return nil, errors.New("watcher stopped")
+	default:
+	}
+
+	// try get the event
+	select {
+	case event, _ := <-w.fw.Events:
+		if event.Op == fsnotify.Rename {
+			// check existence of file, and add watch again
+			_, err := os.Stat(event.Name)
+			if err == nil || os.IsExist(err) {
+				w.fw.Add(event.Name)
+			}
+		}
+
+		c, err := w.f.Read()
+		if err != nil {
+			return nil, err
+		}
+		return c, nil
+	case err := <-w.fw.Errors:
+		return nil, err
+	case <-w.exit:
+		return nil, errors.New("watcher stopped")
+	}
+}
+
+func (w *watcher) Stop() error {
+	return w.fw.Close()
+}

config/source/flag/README.md

@@ -0,0 +1,47 @@
+# Flag Source
+
+The flag source reads config from flags
+
+## Format
+
+We expect the use of the `flag` package. Upper case flags will be lower cased. Dashes will be used as delimiters.
+
+### Example
+
+```
+dbAddress := flag.String("database_address", "127.0.0.1", "the db address")
+dbPort := flag.Int("database_port", 3306, "the db port)
+```
+
+Becomes
+
+```json
+{
+    "database": {
+        "address": "127.0.0.1",
+        "port": 3306
+    }
+}
+```
+
+## New Source
+
+```go
+flagSource := flag.NewSource(
+	// optionally enable reading of unset flags and their default
+	// values into config, defaults to false
+	IncludeUnset(true)
+)
+```
+
+## Load Source
+
+Load the source into config
+
+```go
+// Create new config
+conf := config.NewConfig()
+
+// Load file source
+conf.Load(flagSource)
+```

config/source/flag/flag.go

@@ -0,0 +1,97 @@
+package flag
+
+import (
+	"errors"
+	"flag"
+	"github.com/imdario/mergo"
+	"github.com/micro/go-micro/config/source"
+	"strings"
+	"time"
+)
+
+type flagsrc struct {
+	opts source.Options
+}
+
+func (fs *flagsrc) Read() (*source.ChangeSet, error) {
+	if !flag.Parsed() {
+		return nil, errors.New("flags not parsed")
+	}
+
+	var changes map[string]interface{}
+
+	visitFn := func(f *flag.Flag) {
+		n := strings.ToLower(f.Name)
+		keys := strings.FieldsFunc(n, split)
+		reverse(keys)
+
+		tmp := make(map[string]interface{})
+		for i, k := range keys {
+			if i == 0 {
+				tmp[k] = f.Value
+				continue
+			}
+
+			tmp = map[string]interface{}{k: tmp}
+		}
+
+		mergo.Map(&changes, tmp) // need to sort error handling
+		return
+	}
+
+	unset, ok := fs.opts.Context.Value(includeUnsetKey{}).(bool)
+	if ok && unset {
+		flag.VisitAll(visitFn)
+	} else {
+		flag.Visit(visitFn)
+	}
+
+	b, err := fs.opts.Encoder.Encode(changes)
+	if err != nil {
+		return nil, err
+	}
+
+	cs := &source.ChangeSet{
+		Format:    fs.opts.Encoder.String(),
+		Data:      b,
+		Timestamp: time.Now(),
+		Source:    fs.String(),
+	}
+	cs.Checksum = cs.Sum()
+
+	return cs, nil
+}
+
+func split(r rune) bool {
+	return r == '-' || r == '_'
+}
+
+func reverse(ss []string) {
+	for i := len(ss)/2 - 1; i >= 0; i-- {
+		opp := len(ss) - 1 - i
+		ss[i], ss[opp] = ss[opp], ss[i]
+	}
+}
+
+func (fs *flagsrc) Watch() (source.Watcher, error) {
+	return source.NewNoopWatcher()
+}
+
+func (fs *flagsrc) String() string {
+	return "flag"
+}
+
+// NewSource returns a config source for integrating parsed flags.
+// Hyphens are delimiters for nesting, and all keys are lowercased.
+//
+// Example:
+//      dbhost := flag.String("database-host", "localhost", "the db host name")
+//
+//      {
+//          "database": {
+//              "host": "localhost"
+//          }
+//      }
+func NewSource(opts ...source.Option) source.Source {
+	return &flagsrc{opts: source.NewOptions(opts...)}
+}

config/source/flag/flag_test.go

@@ -0,0 +1,66 @@
+package flag
+
+import (
+	"encoding/json"
+	"flag"
+	"testing"
+)
+
+var (
+	dbuser = flag.String("database-user", "default", "db user")
+	dbhost = flag.String("database-host", "", "db host")
+	dbpw   = flag.String("database-password", "", "db pw")
+)
+
+func init() {
+	flag.Set("database-host", "localhost")
+	flag.Set("database-password", "some-password")
+	flag.Parse()
+}
+
+func TestFlagsrc_Read(t *testing.T) {
+	source := NewSource()
+	c, err := source.Read()
+	if err != nil {
+		t.Error(err)
+	}
+
+	var actual map[string]interface{}
+	if err := json.Unmarshal(c.Data, &actual); err != nil {
+		t.Error(err)
+	}
+
+	actualDB := actual["database"].(map[string]interface{})
+	if actualDB["host"] != *dbhost {
+		t.Errorf("expected %v got %v", *dbhost, actualDB["host"])
+	}
+
+	if actualDB["password"] != *dbpw {
+		t.Errorf("expected %v got %v", *dbpw, actualDB["password"])
+	}
+
+	// unset flags should not be loaded
+	if actualDB["user"] != nil {
+		t.Errorf("expected %v got %v", nil, actualDB["user"])
+	}
+}
+
+func TestFlagsrc_ReadAll(t *testing.T) {
+	source := NewSource(IncludeUnset(true))
+	c, err := source.Read()
+	if err != nil {
+		t.Error(err)
+	}
+
+	var actual map[string]interface{}
+	if err := json.Unmarshal(c.Data, &actual); err != nil {
+		t.Error(err)
+	}
+
+	actualDB := actual["database"].(map[string]interface{})
+
+	// unset flag defaults should be loaded
+	if actualDB["user"] != *dbuser {
+		t.Errorf("expected %v got %v", *dbuser, actualDB["user"])
+	}
+}

config/source/flag/options.go

@@ -0,0 +1,20 @@
+package flag
+
+import (
+	"context"
+
+	"github.com/micro/go-micro/config/source"
+)
+
+type includeUnsetKey struct{}
+
+// IncludeUnset toggles the loading of unset flags and their respective default values.
+// Default behavior is to ignore any unset flags.
+func IncludeUnset(b bool) source.Option {
+	return func(o *source.Options) {
+		if o.Context == nil {
+			o.Context = context.Background()
+		}
+		o.Context = context.WithValue(o.Context, includeUnsetKey{}, true)
+	}
+}

config/source/memory/README.md

@@ -0,0 +1,44 @@
+# Memory Source
+
+The memory source provides in-memory data as a source
+
+## Memory Format
+
+The expected data format is json
+
+```json
+data := []byte(`{
+    "hosts": {
+        "database": {
+            "address": "10.0.0.1",
+            "port": 3306
+        },
+        "cache": {
+            "address": "10.0.0.2",
+            "port": 6379
+        }
+    }
+}`)
+```
+
+## New Source
+
+Specify source with data
+
+```go
+memorySource := memory.NewSource(
+	memory.WithData(data),
+)
+```
+
+## Load Source
+
+Load the source into config
+
+```go
+// Create new config
+conf := config.NewConfig()
+
+// Load file source
+conf.Load(memorySource)
+```

config/source/memory/memory.go

@@ -0,0 +1,93 @@
+// Package memory is a memory source
+package memory
+
+import (
+	"sync"
+	"time"
+
+	"github.com/micro/go-micro/config/source"
+	"github.com/pborman/uuid"
+)
+
+type memory struct {
+	sync.RWMutex
+	ChangeSet *source.ChangeSet
+	Watchers  map[string]*watcher
+}
+
+func (s *memory) Read() (*source.ChangeSet, error) {
+	s.RLock()
+	cs := &source.ChangeSet{
+		Timestamp: s.ChangeSet.Timestamp,
+		Data:      s.ChangeSet.Data,
+		Checksum:  s.ChangeSet.Checksum,
+		Source:    s.ChangeSet.Source,
+	}
+	s.RUnlock()
+	return cs, nil
+}
+
+func (s *memory) Watch() (source.Watcher, error) {
+	w := &watcher{
+		Id:      uuid.NewUUID().String(),
+		Updates: make(chan *source.ChangeSet, 100),
+		Source:  s,
+	}
+
+	s.Lock()
+	s.Watchers[w.Id] = w
+	s.Unlock()
+	return w, nil
+}
+
+// Update allows manual updates of the config data.
+func (s *memory) Update(c *source.ChangeSet) {
+	// don't process nil
+	if c == nil {
+		return
+	}
+
+	// hash the file
+	s.Lock()
+	// update changeset
+	s.ChangeSet = &source.ChangeSet{
+		Data:      c.Data,
+		Format:    c.Format,
+		Source:    "memory",
+		Timestamp: time.Now(),
+	}
+	s.ChangeSet.Checksum = s.ChangeSet.Sum()
+
+	// update watchers
+	for _, w := range s.Watchers {
+		select {
+		case w.Updates <- s.ChangeSet:
+		default:
+		}
+	}
+	s.Unlock()
+}
+
+func (s *memory) String() string {
+	return "memory"
+}
+
+func NewSource(opts ...source.Option) source.Source {
+	var options source.Options
+	for _, o := range opts {
+		o(&options)
+	}
+
+	s := &memory{
+		Watchers: make(map[string]*watcher),
+	}
+
+	if options.Context != nil {
+		c, ok := options.Context.Value(changeSetKey{}).(*source.ChangeSet)
+		if ok {
+			s.Update(c)
+		}
+	}
+
+	return s
+}

config/source/memory/options.go

@@ -0,0 +1,41 @@
+package memory
+
+import (
+	"context"
+
+	"github.com/micro/go-micro/config/source"
+)
+
+type changeSetKey struct{}
+
+func withData(d []byte, f string) source.Option {
+	return func(o *source.Options) {
+		if o.Context == nil {
+			o.Context = context.Background()
+		}
+		o.Context = context.WithValue(o.Context, changeSetKey{}, &source.ChangeSet{
+			Data:   d,
+			Format: f,
+		})
+	}
+}
+
+// WithChangeSet allows a changeset to be set
+func WithChangeSet(cs *source.ChangeSet) source.Option {
+	return func(o *source.Options) {
+		if o.Context == nil {
+			o.Context = context.Background()
+		}
+		o.Context = context.WithValue(o.Context, changeSetKey{}, cs)
+	}
+}
+
+// WithJSON allows the source data to be set to json
+func WithJSON(d []byte) source.Option {
+	return withData(d, "json")
+}
+
+// WithYAML allows the source data to be set to yaml
+func WithYAML(d []byte) source.Option {
+	return withData(d, "yaml")
+}

config/source/memory/watcher.go

@@ -0,0 +1,23 @@
+package memory
+
+import (
+	"github.com/micro/go-micro/config/source"
+)
+
+type watcher struct {
+	Id      string
+	Updates chan *source.ChangeSet
+	Source  *memory
+}
+
+func (w *watcher) Next() (*source.ChangeSet, error) {
+	cs := <-w.Updates
+	return cs, nil
+}
+
+func (w *watcher) Stop() error {
+	w.Source.Lock()
+	delete(w.Source.Watchers, w.Id)
+	w.Source.Unlock()
+	return nil
+}

config/source/noop.go

@@ -0,0 +1,25 @@
+package source
+
+import (
+	"errors"
+)
+
+type noopWatcher struct {
+	exit chan struct{}
+}
+
+func (w *noopWatcher) Next() (*ChangeSet, error) {
+	<-w.exit
+
+	return nil, errors.New("noopWatcher stopped")
+}
+
+func (w *noopWatcher) Stop() error {
+	close(w.exit)
+	return nil
+}
+
+// NewNoopWatcher returns a watcher that blocks on Next() until Stop() is called.
+func NewNoopWatcher() (Watcher, error) {
+	return &noopWatcher{exit: make(chan struct{})}, nil
+}

config/source/options.go

@@ -0,0 +1,38 @@
+package source
+
+import (
+	"context"
+
+	"github.com/micro/go-micro/config/encoder"
+	"github.com/micro/go-micro/config/encoder/json"
+)
+
+type Options struct {
+	// Encoder
+	Encoder encoder.Encoder
+
+	// for alternative data
+	Context context.Context
+}
+
+type Option func(o *Options)
+
+func NewOptions(opts ...Option) Options {
+	options := Options{
+		Encoder: json.NewEncoder(),
+		Context: context.Background(),
+	}
+
+	for _, o := range opts {
+		o(&options)
+	}
+
+	return options
+}
+
+// WithEncoder sets the source encoder
+func WithEncoder(e encoder.Encoder) Option {
+	return func(o *Options) {
+		o.Encoder = e
+	}
+}

config/source/source.go

@@ -0,0 +1,28 @@
+// Package source is the interface for sources
+package source
+
+import (
+	"time"
+)
+
+// Source is the source from which config is loaded
+type Source interface {
+	Read() (*ChangeSet, error)
+	Watch() (Watcher, error)
+	String() string
+}
+
+// ChangeSet represents a set of changes from a source
+type ChangeSet struct {
+	Data      []byte
+	Checksum  string
+	Format    string
+	Source    string
+	Timestamp time.Time
+}
+
+// Watcher watches a source for changes
+type Watcher interface {
+	Next() (*ChangeSet, error)
+	Stop() error
+}

config/value.go

@@ -0,0 +1,49 @@
+package config
+
+import (
+	"time"
+
+	"github.com/micro/go-micro/config/reader"
+)
+
+type value struct{}
+
+func newValue() reader.Value {
+	return new(value)
+}
+
+func (v *value) Bool(def bool) bool {
+	return false
+}
+
+func (v *value) Int(def int) int {
+	return 0
+}
+
+func (v *value) String(def string) string {
+	return ""
+}
+
+func (v *value) Float64(def float64) float64 {
+	return 0.0
+}
+
+func (v *value) Duration(def time.Duration) time.Duration {
+	return time.Duration(0)
+}
+
+func (v *value) StringSlice(def []string) []string {
+	return nil
+}
+
+func (v *value) StringMap(def map[string]string) map[string]string {
+	return map[string]string{}
+}
+
+func (v *value) Scan(val interface{}) error {
+	return nil
+}
+
+func (v *value) Bytes() []byte {
+	return nil
+}

function.go

@@ -54,7 +54,7 @@ func newFunction(opts ...Option) Function {
 
 	service.Server().Init(
 		// ensure the service waits for requests to finish
-		server.Wait(true),
+		server.Wait(nil),
 		// wrap handlers and subscribers to finish execution
 		server.WrapHandler(fnHandlerWrapper(fn)),
 		server.WrapSubscriber(fnSubWrapper(fn)),

go.mod

@@ -1,124 +1,124 @@
 module github.com/micro/go-micro
 
 require (
-	cloud.google.com/go v0.36.0 // indirect
-	github.com/DataDog/datadog-go v0.0.0-20190323183505-07c7c350327b // indirect
-	github.com/Microsoft/go-winio v0.4.12 // indirect
-	github.com/NYTimes/gziphandler v1.1.1 // indirect
-	github.com/SAP/go-hdb v0.14.1 // indirect
-	github.com/StackExchange/wmi v0.0.0-20181212234831-e0a55b97c705 // indirect
-	github.com/aliyun/alibaba-cloud-sdk-go v0.0.0-20190328075325-5887fb5d0027 // indirect
-	github.com/araddon/gou v0.0.0-20190110011759-c797efecbb61 // indirect
-	github.com/aws/aws-sdk-go v1.17.5 // indirect
-	github.com/boombuler/barcode v1.0.0 // indirect
-	github.com/briankassouf/jose v0.9.1 // indirect
-	github.com/cenkalti/backoff v2.1.1+incompatible // indirect
-	github.com/centrify/cloud-golang-sdk v0.0.0-20190214225812-119110094d0f // indirect
-	github.com/chrismalek/oktasdk-go v0.0.0-20181212195951-3430665dfaa0 // indirect
-	github.com/circonus-labs/circonus-gometrics v2.2.6+incompatible // indirect
-	github.com/containerd/continuity v0.0.0-20181203112020-004b46473808 // indirect
-	github.com/coredns/coredns v1.4.0 // indirect
-	github.com/coreos/etcd v3.3.12+incompatible // indirect
-	github.com/coreos/go-oidc v2.0.0+incompatible // indirect
-	github.com/coreos/go-semver v0.2.0 // indirect
+	cloud.google.com/go v0.39.0 // indirect
+	github.com/BurntSushi/toml v0.3.1
+	github.com/OneOfOne/xxhash v1.2.5 // indirect
+	github.com/armon/circbuf v0.0.0-20190214190532-5111143e8da2 // indirect
+	github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 // indirect
+	github.com/armon/go-radix v1.0.0 // indirect
+	github.com/beevik/ntp v0.2.0
+	github.com/bitly/go-simplejson v0.5.0
+	github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 // indirect
+	github.com/bradfitz/gomemcache v0.0.0-20190329173943-551aad21a668
+	github.com/bwmarrin/discordgo v0.19.0
+	github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc // indirect
+	github.com/coreos/bbolt v1.3.2 // indirect
+	github.com/coreos/etcd v3.3.13+incompatible
+	github.com/coreos/go-semver v0.3.0 // indirect
+	github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e // indirect
 	github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f // indirect
-	github.com/dancannon/gorethink v4.0.0+incompatible // indirect
-	github.com/denisenkom/go-mssqldb v0.0.0-20190328043727-2183450503ad // indirect
-	github.com/dimchansky/utfbom v1.1.0 // indirect
-	github.com/duosecurity/duo_api_golang v0.0.0-20190308151101-6c680f768e74 // indirect
-	github.com/envoyproxy/go-control-plane v0.6.9 // indirect
-	github.com/fullsailor/pkcs7 v0.0.0-20180613152042-8306686428a5 // indirect
-	github.com/gammazero/deque v0.0.0-20190130191400-2afb3858e9c7 // indirect
-	github.com/gammazero/workerpool v0.0.0-20181230203049-86a96b5d5d92 // indirect
+	github.com/dgrijalva/jwt-go v3.2.0+incompatible // indirect
+	github.com/dgryski/go-sip13 v0.0.0-20190329191031-25c5027a8c7b // indirect
+	github.com/emirpasic/gods v1.12.0 // indirect
+	github.com/forestgiant/sliceutil v0.0.0-20160425183142-94783f95db6c
+	github.com/fsnotify/fsnotify v1.4.7
+	github.com/fsouza/go-dockerclient v1.4.1
 	github.com/garyburd/redigo v1.6.0 // indirect
-	github.com/go-errors/errors v1.0.1 // indirect
-	github.com/go-ldap/ldap v3.0.2+incompatible // indirect
+	github.com/ghodss/yaml v1.0.0
+	github.com/gliderlabs/ssh v0.1.4 // indirect
 	github.com/go-log/log v0.1.0
-	github.com/go-ole/go-ole v1.2.4 // indirect
-	github.com/go-sql-driver/mysql v1.4.1 // indirect
-	github.com/go-stomp/stomp v2.0.2+incompatible // indirect
-	github.com/gocql/gocql v0.0.0-20190325140904-fc3925ac2cbd // indirect
-	github.com/gogo/googleapis v1.1.0 // indirect
-	github.com/gogo/protobuf v1.2.1 // indirect
+	github.com/go-redsync/redsync v1.2.0
+	github.com/go-telegram-bot-api/telegram-bot-api v4.6.4+incompatible // indirect
 	github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef // indirect
+	github.com/golang/mock v1.3.1 // indirect
 	github.com/golang/protobuf v1.3.1
-	github.com/golang/snappy v0.0.1 // indirect
+	github.com/gomodule/redigo v2.0.0+incompatible
 	github.com/google/btree v1.0.0 // indirect
+	github.com/google/pprof v0.0.0-20190515194954-54271f7e092f // indirect
 	github.com/google/uuid v1.1.1
-	github.com/gorhill/cronexpr v0.0.0-20180427100037-88b0669f7d75 // indirect
-	github.com/gorilla/websocket v1.4.0 // indirect
 	github.com/grpc-ecosystem/go-grpc-middleware v1.0.0 // indirect
 	github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect
-	github.com/grpc-ecosystem/grpc-gateway v1.8.5 // indirect
-	github.com/hashicorp/consul v1.4.4
-	github.com/hashicorp/go-discover v0.0.0-20190319153616-61771d82ff54 // indirect
-	github.com/hashicorp/go-gcp-common v0.5.0 // indirect
-	github.com/hashicorp/go-hclog v0.8.0 // indirect
-	github.com/hashicorp/go-memdb v1.0.0 // indirect
-	github.com/hashicorp/go-plugin v1.0.0 // indirect
-	github.com/hashicorp/go-retryablehttp v0.5.3 // indirect
-	github.com/hashicorp/go-rootcerts v1.0.0 // indirect
+	github.com/grpc-ecosystem/grpc-gateway v1.9.0 // indirect
+	github.com/hashicorp/consul/api v1.1.0
+	github.com/hashicorp/go-immutable-radix v1.1.0 // indirect
+	github.com/hashicorp/go-msgpack v0.5.5 // indirect
 	github.com/hashicorp/go-sockaddr v1.0.2 // indirect
-	github.com/hashicorp/go-version v1.1.0 // indirect
-	github.com/hashicorp/golang-lru v0.5.1 // indirect
-	github.com/hashicorp/hil v0.0.0-20190212132231-97b3a9cdfa93 // indirect
-	github.com/hashicorp/memberlist v0.1.3
-	github.com/hashicorp/net-rpc-msgpackrpc v0.0.0-20151116020338-a14192a58a69 // indirect
-	github.com/hashicorp/nomad v0.8.7 // indirect
-	github.com/hashicorp/raft-boltdb v0.0.0-20171010151810-6e5ba93211ea // indirect
-	github.com/hashicorp/serf v0.8.2 // indirect
-	github.com/hashicorp/vault v1.1.0 // indirect
-	github.com/hashicorp/vault-plugin-auth-alicloud v0.0.0-20190320211238-36e70c54375f // indirect
-	github.com/hashicorp/vault-plugin-auth-azure v0.0.0-20190320211138-f34b96803f04 // indirect
-	github.com/hashicorp/vault-plugin-auth-centrify v0.0.0-20190320211357-44eb061bdfd8 // indirect
-	github.com/hashicorp/vault-plugin-auth-gcp v0.0.0-20190320214413-e8308b5e41c9 // indirect
-	github.com/hashicorp/vault-plugin-auth-jwt v0.0.0-20190321042813-9474f90fb1df // indirect
-	github.com/hashicorp/vault-plugin-auth-kubernetes v0.0.0-20190328163920-79931ee7aad5 // indirect
-	github.com/hashicorp/vault-plugin-secrets-ad v0.0.0-20190327182327-ed2c3d4c3d95 // indirect
-	github.com/hashicorp/vault-plugin-secrets-alicloud v0.0.0-20190320213517-3307bdf683cb // indirect
-	github.com/hashicorp/vault-plugin-secrets-azure v0.0.0-20190320211922-2dc8a8a5e490 // indirect
-	github.com/hashicorp/vault-plugin-secrets-gcp v0.0.0-20190320211452-71903323ecb4 // indirect
-	github.com/hashicorp/vault-plugin-secrets-gcpkms v0.0.0-20190320213325-9e326a9e802d // indirect
-	github.com/hashicorp/vault-plugin-secrets-kv v0.0.0-20190320211621-3ccc8684cf25 // indirect
-	github.com/influxdata/influxdb v1.7.5 // indirect
-	github.com/influxdata/platform v0.0.0-20190117200541-d500d3cf5589 // indirect
-	github.com/jeffchao/backoff v0.0.0-20140404060208-9d7fd7aa17f2 // indirect
-	github.com/jefferai/jsonx v1.0.0 // indirect
+	github.com/hashicorp/hcl v1.0.0
+	github.com/hashicorp/mdns v1.0.1 // indirect
+	github.com/hashicorp/memberlist v0.1.4
+	github.com/hashicorp/serf v0.8.3 // indirect
+	github.com/imdario/mergo v0.3.7
 	github.com/jonboulle/clockwork v0.1.0 // indirect
-	github.com/keybase/go-crypto v0.0.0-20190312101036-b475f2ecc1fe // indirect
-	github.com/lyft/protoc-gen-validate v0.0.14 // indirect
-	github.com/mattbaird/elastigo v0.0.0-20170123220020-2fe47fd29e4b // indirect
-	github.com/mattn/go-isatty v0.0.7 // indirect
-	github.com/michaelklishin/rabbit-hole v1.5.0 // indirect
-	github.com/micro/cli v0.1.0
-	github.com/micro/go-log v0.1.0
-	github.com/micro/go-rcache v0.3.0
+	github.com/json-iterator/go v1.1.6 // indirect
+	github.com/kisielk/errcheck v1.2.0 // indirect
+	github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect
+	github.com/kr/pty v1.1.4 // indirect
+	github.com/lusis/go-slackbot v0.0.0-20180109053408-401027ccfef5 // indirect
+	github.com/lusis/slack-test v0.0.0-20190426140909-c40012f20018 // indirect
+	github.com/mattn/go-colorable v0.1.2 // indirect
+	github.com/micro/cli v0.2.0
 	github.com/micro/mdns v0.1.0
-	github.com/micro/util v0.2.0
+	github.com/miekg/dns v1.1.13 // indirect
+	github.com/mitchellh/go-homedir v1.1.0 // indirect
 	github.com/mitchellh/hashstructure v1.0.0
-	github.com/mitchellh/pointerstructure v0.0.0-20190323210102-2db4bb651397 // indirect
-	github.com/ory-am/common v0.4.0 // indirect
-	github.com/ory/dockertest v3.3.4+incompatible // indirect
-	github.com/pascaldekloe/goe v0.1.0 // indirect
-	github.com/pborman/uuid v1.2.0 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
+	github.com/modern-go/reflect2 v1.0.1 // indirect
+	github.com/nats-io/gnatsd v1.4.1 // indirect
+	github.com/nats-io/go-nats v1.7.2 // indirect
+	github.com/nats-io/nats.go v1.7.2
+	github.com/nats-io/nkeys v0.0.2 // indirect
+	github.com/nats-io/nuid v1.0.1 // indirect
+	github.com/nlopes/slack v0.5.0
+	github.com/onsi/ginkgo v1.8.0 // indirect
+	github.com/onsi/gomega v1.5.0 // indirect
+	github.com/pborman/uuid v1.2.0
 	github.com/pkg/errors v0.8.1
-	github.com/pquerna/cachecontrol v0.0.0-20180517163645-1555304b9b35 // indirect
-	github.com/pquerna/otp v1.1.0 // indirect
-	github.com/ryanuber/go-glob v1.0.0 // indirect
-	github.com/samuel/go-zookeeper v0.0.0-20180130194729-c4fab1ac1bec // indirect
-	github.com/shirou/gopsutil v2.18.12+incompatible // indirect
-	github.com/shirou/w32 v0.0.0-20160930032740-bb4de0191aa4 // indirect
+	github.com/posener/complete v1.2.1 // indirect
+	github.com/prometheus/client_golang v0.9.3 // indirect
+	github.com/prometheus/common v0.4.1 // indirect
+	github.com/prometheus/procfs v0.0.1 // indirect
+	github.com/prometheus/tsdb v0.8.0 // indirect
+	github.com/rogpeppe/fastuuid v1.1.0 // indirect
+	github.com/sirupsen/logrus v1.4.2 // indirect
 	github.com/soheilhy/cmux v0.1.4 // indirect
-	github.com/streadway/amqp v0.0.0-20190312223743-14f78b41ce6d // indirect
+	github.com/spaolacci/murmur3 v1.1.0 // indirect
+	github.com/stretchr/objx v0.2.0 // indirect
+	github.com/technoweenie/multipartstreamer v1.0.1 // indirect
 	github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 // indirect
-	github.com/ugorji/go/codec v0.0.0-20190320090025-2dc34c0b8780 // indirect
+	github.com/xanzy/ssh-agent v0.2.1 // indirect
 	github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect
-	golang.org/x/net v0.0.0-20190327214358-63eda1eb0650
+	go.etcd.io/bbolt v1.3.2 // indirect
+	go.etcd.io/etcd v3.3.13+incompatible
+	go.opencensus.io v0.22.0 // indirect
+	go.uber.org/atomic v1.4.0 // indirect
+	go.uber.org/multierr v1.1.0 // indirect
+	go.uber.org/zap v1.10.0 // indirect
+	golang.org/x/crypto v0.0.0-20190530122614-20be4c3c3ed5 // indirect
+	golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522 // indirect
+	golang.org/x/image v0.0.0-20190523035834-f03afa92d3ff // indirect
+	golang.org/x/lint v0.0.0-20190409202823-959b441ac422 // indirect
+	golang.org/x/mobile v0.0.0-20190509164839-32b2708ab171 // indirect
+	golang.org/x/net v0.0.0-20190522155817-f3200d17e092
+	golang.org/x/oauth2 v0.0.0-20190523182746-aaccbc9213b0 // indirect
+	golang.org/x/sys v0.0.0-20190531132440-69e3a3a65b5b // indirect
 	golang.org/x/time v0.0.0-20190308202827-9d24e82272b4 // indirect
-	google.golang.org/grpc v1.19.1 // indirect
-	gopkg.in/asn1-ber.v1 v1.0.0-20181015200546-f715ec2f112d // indirect
-	gopkg.in/gorethink/gorethink.v4 v4.1.0 // indirect
-	gopkg.in/ini.v1 v1.42.0 // indirect
-	gopkg.in/ory-am/dockertest.v2 v2.2.3 // indirect
-	layeh.com/radius v0.0.0-20190322222518-890bc1058917 // indirect
+	golang.org/x/tools v0.0.0-20190530215528-75312fb06703 // indirect
+	google.golang.org/appengine v1.6.0 // indirect
+	google.golang.org/genproto v0.0.0-20190530194941-fb225487d101 // indirect
+	google.golang.org/grpc v1.21.0 // indirect
+	gopkg.in/bsm/ratelimit.v1 v1.0.0-20160220154919-db14e161995a // indirect
+	gopkg.in/redis.v3 v3.6.4
+	gopkg.in/src-d/go-billy.v4 v4.3.0 // indirect
+	gopkg.in/src-d/go-git-fixtures.v3 v3.5.0 // indirect
+	gopkg.in/src-d/go-git.v4 v4.11.0
+	gopkg.in/telegram-bot-api.v4 v4.6.4
+	honnef.co/go/tools v0.0.0-20190530170028-a1efa522b896 // indirect
+)
+
+exclude sourcegraph.com/sourcegraph/go-diff v0.5.1
+
+replace (
+	github.com/golang/lint => github.com/golang/lint v0.0.0-20190227174305-8f45f776aaf1
+	github.com/testcontainers/testcontainer-go => github.com/testcontainers/testcontainers-go v0.0.0-20181115231424-8e868ca12c0f
 )

publisher.go

@@ -12,5 +12,5 @@ type publisher struct {
 }
 
 func (p *publisher) Publish(ctx context.Context, msg interface{}, opts ...client.PublishOption) error {
-	return p.c.Publish(ctx, p.c.NewMessage(p.topic, msg))
+	return p.c.Publish(ctx, p.c.NewMessage(p.topic, msg), opts...)
 }

registry/cache/README.md

@@ -0,0 +1,31 @@
+# Registry Cache 
+
+Cache is a library that provides a caching layer for the go-micro [registry](https://godoc.org/github.com/micro/go-micro/registry#Registry).
+
+If you're looking for caching in your microservices use the [selector](https://micro.mu/docs/fault-tolerance.html#caching-discovery).
+
+## Interface
+
+```
+// Cache is the registry cache interface
+type Cache interface {
+	// embed the registry interface
+	registry.Registry
+	// stop the cache watcher
+	Stop()
+}
+```
+
+## Usage
+
+```
+import (
+	"github.com/micro/go-micro/registry"
+	"github.com/micro/go-micro/registry/cache"
+)
+
+r := registry.NewRegistry()
+cache := cache.New(r)
+
+services, _ := cache.GetService("my.service")
+```