Merge pull request #490 from micro/stdlib

The move to a standard library for microservices

.travis.yml

@@ -2,6 +2,8 @@ language: go
 go:
 - 1.11.x
 - 1.12.x
+env:
+  - GO111MODULE=on
 notifications:
   slack:
     secure: aEvhLbhujaGaKSrOokiG3//PaVHTIrc3fBpoRbCRqfZpyq6WREoapJJhF+tIpWWOwaC9GmChbD6aHo/jMUgwKXVyPSaNjiEL87YzUUpL8B2zslNp1rgfTg/LrzthOx3Q1TYwpaAl3to0fuHUVFX4yMeC2vuThq7WSXgMMxFCtbc=

agent/README.md

@@ -0,0 +1,197 @@
+# Agent 
+
+Agent is a library used to create commands, inputs and robot services
+
+## Getting Started
+
+- [Commands](#commands) - Commands are functions executed by the bot based on text based pattern matching.
+- [Inputs](#inputs) - Inputs are plugins for communication e.g Slack, Telegram, IRC, etc.
+- [Services](#services) - Write bots as micro services
+
+## Commands
+
+Commands are functions executed by the bot based on text based pattern matching.
+
+### Write a Command
+
+```go
+import "github.com/micro/go-micro/agent/command"
+
+func Ping() command.Command {
+	usage := "ping"
+	description := "Returns pong"
+
+	return command.NewCommand("ping", usage, desc, func(args ...string) ([]byte, error) {
+		return []byte("pong"), nil
+	})
+}
+```
+
+### Register the command
+
+Add the command to the Commands map with a pattern key that can be matched by golang/regexp.Match
+
+```go
+import "github.com/micro/go-micro/agent/command"
+
+func init() {
+	command.Commands["^ping$"] = Ping()
+}
+```
+
+### Rebuild Micro
+
+Build binary
+```shell
+cd github.com/micro/micro
+
+// For local use
+go build -i -o micro ./main.go
+
+// For docker image
+CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-w' -i -o micro ./main.go
+```
+
+## Inputs
+
+Inputs are plugins for communication e.g Slack, HipChat, XMPP, IRC, SMTP, etc, etc. 
+
+New inputs can be added in the following way.
+
+### Write an Input
+
+Write an input that satisfies the Input interface.
+
+```go
+type Input interface {
+	// Provide cli flags
+	Flags() []cli.Flag
+	// Initialise input using cli context
+	Init(*cli.Context) error
+	// Stream events from the input
+	Stream() (Conn, error)
+	// Start the input
+	Start() error
+	// Stop the input
+	Stop() error
+	// name of the input
+	String() string
+}
+```
+
+### Register the input
+
+Add the input to the Inputs map.
+
+```go
+import "github.com/micro/micro/bot/input"
+
+func init() {
+	input.Inputs["name"] = MyInput
+}
+```
+
+### Rebuild Micro
+
+Build binary
+```shell
+cd github.com/micro/micro
+
+// For local use
+go build -i -o micro ./main.go
+
+// For docker image
+CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -ldflags '-w' -i -o micro ./main.go
+```
+
+## Services
+
+The micro bot supports the ability to create commands as micro services. 
+
+### How does it work?
+
+The bot watches the service registry for services with it's namespace. The default namespace is `go.micro.bot`. 
+Any service within this namespace will automatically be added to the list of available commands. When a command 
+is executed, the bot will call the service with method `Command.Exec`. It also expects the method `Command.Help` 
+to exist for usage info.
+
+
+The service interface is as follows and can be found at [go-micro/agent/proto](https://github.com/micro/go-micro/agent/blob/master/proto/bot.proto)
+
+```
+syntax = "proto3";
+
+package go.micro.bot;
+
+service Command {
+	rpc Help(HelpRequest) returns (HelpResponse) {};
+	rpc Exec(ExecRequest) returns (ExecResponse) {};
+}
+
+message HelpRequest {
+}
+
+message HelpResponse {
+	string usage = 1;
+	string description = 2;
+}
+
+message ExecRequest {
+	repeated string args = 1;
+}
+
+message ExecResponse {
+	bytes result = 1;
+	string error = 2;
+}
+```
+
+### Example
+
+Here's an example echo command as a microservice
+
+```go
+package main
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/micro/go-micro"
+	"golang.org/x/net/context"
+
+	proto "github.com/micro/go-micro/agent/proto"
+)
+
+type Command struct{}
+
+// Help returns the command usage
+func (c *Command) Help(ctx context.Context, req *proto.HelpRequest, rsp *proto.HelpResponse) error {
+	// Usage should include the name of the command
+	rsp.Usage = "echo"
+	rsp.Description = "This is an example bot command as a micro service which echos the message"
+	return nil
+}
+
+// Exec executes the command
+func (c *Command) Exec(ctx context.Context, req *proto.ExecRequest, rsp *proto.ExecResponse) error {
+	rsp.Result = []byte(strings.Join(req.Args, " "))
+	// rsp.Error could be set to return an error instead
+	// the function error would only be used for service level issues
+	return nil
+}
+
+func main() {
+	service := micro.NewService(
+		micro.Name("go.micro.bot.echo"),
+	)
+
+	service.Init()
+
+	proto.RegisterCommandHandler(service.Server(), new(Command))
+
+	if err := service.Run(); err != nil {
+		fmt.Println(err)
+	}
+}
+```

agent/agent.go

@@ -0,0 +1,2 @@
+// Package agent provides an interface for building robots
+package agent

agent/command/command.go

@@ -0,0 +1,54 @@
+// Package command is an interface for defining bot commands
+package command
+
+var (
+	// Commmands keyed by golang/regexp patterns
+	// regexp.Match(key, input) is used to match
+	Commands = map[string]Command{}
+)
+
+// Command is the interface for specific named
+// commands executed via plugins or the bot.
+type Command interface {
+	// Executes the command with args passed in
+	Exec(args ...string) ([]byte, error)
+	// Usage of the command
+	Usage() string
+	// Description of the command
+	Description() string
+	// Name of the command
+	String() string
+}
+
+type cmd struct {
+	name        string
+	usage       string
+	description string
+	exec        func(args ...string) ([]byte, error)
+}
+
+func (c *cmd) Description() string {
+	return c.description
+}
+
+func (c *cmd) Exec(args ...string) ([]byte, error) {
+	return c.exec(args...)
+}
+
+func (c *cmd) Usage() string {
+	return c.usage
+}
+
+func (c *cmd) String() string {
+	return c.name
+}
+
+// NewCommand helps quickly create a new command
+func NewCommand(name, usage, description string, exec func(args ...string) ([]byte, error)) Command {
+	return &cmd{
+		name:        name,
+		usage:       usage,
+		description: description,
+		exec:        exec,
+	}
+}

agent/command/command_test.go

@@ -0,0 +1,65 @@
+package command
+
+import (
+	"testing"
+)
+
+func TestCommand(t *testing.T) {
+	c := &cmd{
+		name:        "test",
+		usage:       "test usage",
+		description: "test description",
+		exec: func(args ...string) ([]byte, error) {
+			return []byte("test"), nil
+		},
+	}
+
+	if c.String() != c.name {
+		t.Fatalf("expected name %s got %s", c.name, c.String())
+	}
+
+	if c.Usage() != c.usage {
+		t.Fatalf("expected usage %s got %s", c.usage, c.Usage())
+	}
+
+	if c.Description() != c.description {
+		t.Fatalf("expected description %s got %s", c.description, c.Description())
+	}
+
+	if r, err := c.Exec(); err != nil {
+		t.Fatal(err)
+	} else if string(r) != "test" {
+		t.Fatalf("expected exec result test got %s", string(r))
+	}
+}
+
+func TestNewCommand(t *testing.T) {
+	c := &cmd{
+		name:        "test",
+		usage:       "test usage",
+		description: "test description",
+		exec: func(args ...string) ([]byte, error) {
+			return []byte("test"), nil
+		},
+	}
+
+	nc := NewCommand(c.name, c.usage, c.description, c.exec)
+
+	if nc.String() != c.name {
+		t.Fatalf("expected name %s got %s", c.name, nc.String())
+	}
+
+	if nc.Usage() != c.usage {
+		t.Fatalf("expected usage %s got %s", c.usage, nc.Usage())
+	}
+
+	if nc.Description() != c.description {
+		t.Fatalf("expected description %s got %s", c.description, nc.Description())
+	}
+
+	if r, err := nc.Exec(); err != nil {
+		t.Fatal(err)
+	} else if string(r) != "test" {
+		t.Fatalf("expected exec result test got %s", string(r))
+	}
+}

agent/input/discord/README.md

@@ -0,0 +1,22 @@
+# Discord input for micro-bot
+[Discord](https://discordapp.com) support for micro bot based on [discordgo](github.com/bwmarrin/discordgo).
+
+This was originally written by Aleksandr Tihomirov (@zet4) and can be found at https://github.com/zet4/micro-misc/.
+
+## Options
+### discord_token
+
+You have to supply an application token via `--discord_token`.
+
+Head over to Discord's [developer introduction](https://discordapp.com/developers/docs/intro)
+to learn how to create applications and how the API works.
+
+### discord_prefix
+
+Set a command prefix with `--discord_prefix`. The default prefix is `Micro `.
+You can mention the bot or use the prefix to run a command.
+
+### discord_whitelist
+
+Pass a list of comma-separated user IDs with `--discord_whitelist`. Only allow
+these users to use the bot.

agent/input/discord/conn.go

@@ -0,0 +1,94 @@
+package discord
+
+import (
+	"errors"
+	"strings"
+	"sync"
+
+	"github.com/bwmarrin/discordgo"
+	"github.com/micro/go-micro/agent/input"
+	"github.com/micro/go-micro/util/log"
+)
+
+type discordConn struct {
+	master *discordInput
+	exit   chan struct{}
+	recv   chan *discordgo.Message
+
+	sync.Mutex
+}
+
+func newConn(master *discordInput) *discordConn {
+	conn := &discordConn{
+		master: master,
+		exit:   make(chan struct{}),
+		recv:   make(chan *discordgo.Message),
+	}
+
+	conn.master.session.AddHandler(func(s *discordgo.Session, m *discordgo.MessageCreate) {
+		if m.Author.ID == master.botID {
+			return
+		}
+
+		whitelisted := false
+		for _, ID := range conn.master.whitelist {
+			if m.Author.ID == ID {
+				whitelisted = true
+				break
+			}
+		}
+
+		if len(master.whitelist) > 0 && !whitelisted {
+			return
+		}
+
+		var valid bool
+		m.Message.Content, valid = conn.master.prefixfn(m.Message.Content)
+		if !valid {
+			return
+		}
+
+		conn.recv <- m.Message
+	})
+
+	return conn
+}
+
+func (dc *discordConn) Recv(event *input.Event) error {
+	for {
+		select {
+		case <-dc.exit:
+			return errors.New("connection closed")
+		case msg := <-dc.recv:
+
+			event.From = msg.ChannelID + ":" + msg.Author.ID
+			event.To = dc.master.botID
+			event.Type = input.TextEvent
+			event.Data = []byte(msg.Content)
+			return nil
+		}
+	}
+}
+
+func (dc *discordConn) Send(e *input.Event) error {
+	fields := strings.Split(e.To, ":")
+	_, err := dc.master.session.ChannelMessageSend(fields[0], string(e.Data))
+	if err != nil {
+		log.Log("[bot][loop][send]", err)
+	}
+	return nil
+}
+
+func (dc *discordConn) Close() error {
+	if err := dc.master.session.Close(); err != nil {
+		return err
+	}
+
+	select {
+	case <-dc.exit:
+		return nil
+	default:
+		close(dc.exit)
+	}
+	return nil
+}

agent/input/discord/discord.go

@@ -0,0 +1,153 @@
+package discord
+
+import (
+	"fmt"
+	"sync"
+
+	"errors"
+	"strings"
+
+	"github.com/bwmarrin/discordgo"
+	"github.com/micro/cli"
+	"github.com/micro/go-micro/agent/input"
+)
+
+func init() {
+	input.Inputs["discord"] = newInput()
+}
+
+func newInput() *discordInput {
+	return &discordInput{}
+}
+
+type discordInput struct {
+	token     string
+	whitelist []string
+	prefix    string
+	prefixfn  func(string) (string, bool)
+	botID     string
+
+	session *discordgo.Session
+
+	sync.Mutex
+	running bool
+	exit    chan struct{}
+}
+
+func (d *discordInput) Flags() []cli.Flag {
+	return []cli.Flag{
+		cli.StringFlag{
+			Name:   "discord_token",
+			EnvVar: "MICRO_DISCORD_TOKEN",
+			Usage:  "Discord token (prefix with Bot if it's for bot account)",
+		},
+		cli.StringFlag{
+			Name:   "discord_whitelist",
+			EnvVar: "MICRO_DISCORD_WHITELIST",
+			Usage:  "Discord Whitelist (seperated by ,)",
+		},
+		cli.StringFlag{
+			Name:   "discord_prefix",
+			Usage:  "Discord Prefix",
+			EnvVar: "MICRO_DISCORD_PREFIX",
+			Value:  "Micro ",
+		},
+	}
+}
+
+func (d *discordInput) Init(ctx *cli.Context) error {
+	token := ctx.String("discord_token")
+	whitelist := ctx.String("discord_whitelist")
+	prefix := ctx.String("discord_prefix")
+
+	if len(token) == 0 {
+		return errors.New("require token")
+	}
+
+	d.token = token
+	d.prefix = prefix
+
+	if len(whitelist) > 0 {
+		d.whitelist = strings.Split(whitelist, ",")
+	}
+
+	return nil
+}
+
+func (d *discordInput) Start() error {
+	if len(d.token) == 0 {
+		return errors.New("missing discord configuration")
+	}
+
+	d.Lock()
+	defer d.Unlock()
+
+	if d.running {
+		return nil
+	}
+
+	var err error
+	d.session, err = discordgo.New(d.token)
+	if err != nil {
+		return err
+	}
+
+	u, err := d.session.User("@me")
+	if err != nil {
+		return err
+	}
+
+	d.botID = u.ID
+	d.prefixfn = CheckPrefixFactory(fmt.Sprintf("<@%s> ", d.botID), fmt.Sprintf("<@!%s> ", d.botID), d.prefix)
+
+	d.exit = make(chan struct{})
+	d.running = true
+
+	return nil
+}
+
+func (d *discordInput) Stream() (input.Conn, error) {
+	d.Lock()
+	defer d.Unlock()
+	if !d.running {
+		return nil, errors.New("not running")
+	}
+
+	//Fire-n-forget close just in case...
+	d.session.Close()
+
+	conn := newConn(d)
+	if err := d.session.Open(); err != nil {
+		return nil, err
+	}
+	return conn, nil
+}
+
+func (d *discordInput) Stop() error {
+	d.Lock()
+	defer d.Unlock()
+
+	if !d.running {
+		return nil
+	}
+
+	close(d.exit)
+	d.running = false
+	return nil
+}
+
+func (d *discordInput) String() string {
+	return "discord"
+}
+
+// CheckPrefixFactory Creates a prefix checking function and stuff.
+func CheckPrefixFactory(prefixes ...string) func(string) (string, bool) {
+	return func(content string) (string, bool) {
+		for _, prefix := range prefixes {
+			if strings.HasPrefix(content, prefix) {
+				return strings.TrimPrefix(content, prefix), true
+			}
+		}
+		return "", false
+	}
+}

agent/input/input.go

@@ -0,0 +1,55 @@
+// Package input is an interface for bot inputs
+package input
+
+import (
+	"github.com/micro/cli"
+)
+
+type EventType string
+
+const (
+	TextEvent EventType = "text"
+)
+
+var (
+	// Inputs keyed by name
+	// Example slack or hipchat
+	Inputs = map[string]Input{}
+)
+
+// Event is the unit sent and received
+type Event struct {
+	Type EventType
+	From string
+	To   string
+	Data []byte
+	Meta map[string]interface{}
+}
+
+// Input is an interface for sources which
+// provide a way to communicate with the bot.
+// Slack, HipChat, XMPP, etc.
+type Input interface {
+	// Provide cli flags
+	Flags() []cli.Flag
+	// Initialise input using cli context
+	Init(*cli.Context) error
+	// Stream events from the input
+	Stream() (Conn, error)
+	// Start the input
+	Start() error
+	// Stop the input
+	Stop() error
+	// name of the input
+	String() string
+}
+
+// Conn interface provides a way to
+// send and receive events. Send and
+// Recv both block until succeeding
+// or failing.
+type Conn interface {
+	Close() error
+	Recv(*Event) error
+	Send(*Event) error
+}

agent/input/slack/conn.go

@@ -0,0 +1,160 @@
+package slack
+
+import (
+	"errors"
+	"fmt"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/micro/go-micro/agent/input"
+	"github.com/nlopes/slack"
+)
+
+// Satisfies the input.Conn interface
+type slackConn struct {
+	auth *slack.AuthTestResponse
+	rtm  *slack.RTM
+	exit chan bool
+
+	sync.Mutex
+	names map[string]string
+}
+
+func (s *slackConn) run() {
+	// func retrieves user names and maps to IDs
+	setNames := func() {
+		names := make(map[string]string)
+		users, err := s.rtm.Client.GetUsers()
+		if err != nil {
+			return
+		}
+
+		for _, user := range users {
+			names[user.ID] = user.Name
+		}
+
+		s.Lock()
+		s.names = names
+		s.Unlock()
+	}
+
+	setNames()
+
+	t := time.NewTicker(time.Minute)
+	defer t.Stop()
+
+	for {
+		select {
+		case <-s.exit:
+			return
+		case <-t.C:
+			setNames()
+		}
+	}
+}
+
+func (s *slackConn) getName(id string) string {
+	s.Lock()
+	name := s.names[id]
+	s.Unlock()
+	return name
+}
+
+func (s *slackConn) Close() error {
+	select {
+	case <-s.exit:
+		return nil
+	default:
+		close(s.exit)
+	}
+	return nil
+}
+
+func (s *slackConn) Recv(event *input.Event) error {
+	if event == nil {
+		return errors.New("event cannot be nil")
+	}
+
+	for {
+		select {
+		case <-s.exit:
+			return errors.New("connection closed")
+		case e := <-s.rtm.IncomingEvents:
+			switch ev := e.Data.(type) {
+			case *slack.MessageEvent:
+				// only accept type message
+				if ev.Type != "message" {
+					continue
+				}
+
+				// only accept DMs or messages to me
+				switch {
+				case strings.HasPrefix(ev.Channel, "D"):
+				case strings.HasPrefix(ev.Text, s.auth.User):
+				case strings.HasPrefix(ev.Text, fmt.Sprintf("<@%s>", s.auth.UserID)):
+				default:
+					continue
+				}
+
+				// Strip username from text
+				switch {
+				case strings.HasPrefix(ev.Text, s.auth.User):
+					args := strings.Split(ev.Text, " ")[1:]
+					ev.Text = strings.Join(args, " ")
+					event.To = s.auth.User
+				case strings.HasPrefix(ev.Text, fmt.Sprintf("<@%s>", s.auth.UserID)):
+					args := strings.Split(ev.Text, " ")[1:]
+					ev.Text = strings.Join(args, " ")
+					event.To = s.auth.UserID
+				}
+
+				if event.Meta == nil {
+					event.Meta = make(map[string]interface{})
+				}
+
+				// fill in the blanks
+				event.From = ev.Channel + ":" + ev.User
+				event.Type = input.TextEvent
+				event.Data = []byte(ev.Text)
+				event.Meta["reply"] = ev
+				return nil
+			case *slack.InvalidAuthEvent:
+				return errors.New("invalid credentials")
+			}
+		}
+	}
+}
+
+func (s *slackConn) Send(event *input.Event) error {
+	var channel, message, name string
+
+	if len(event.To) == 0 {
+		return errors.New("require Event.To")
+	}
+
+	parts := strings.Split(event.To, ":")
+
+	if len(parts) == 2 {
+		channel = parts[0]
+		name = s.getName(parts[1])
+		// try using reply meta
+	} else if ev, ok := event.Meta["reply"]; ok {
+		channel = ev.(*slack.MessageEvent).Channel
+		name = s.getName(ev.(*slack.MessageEvent).User)
+	}
+
+	// don't know where to send the message
+	if len(channel) == 0 {
+		return errors.New("could not determine who message is to")
+	}
+
+	if len(name) == 0 || strings.HasPrefix(channel, "D") {
+		message = string(event.Data)
+	} else {
+		message = fmt.Sprintf("@%s: %s", name, string(event.Data))
+	}
+
+	s.rtm.SendMessage(s.rtm.NewOutgoingMessage(message, channel))
+	return nil
+}

agent/input/slack/slack.go

@@ -0,0 +1,147 @@
+package slack
+
+import (
+	"errors"
+	"sync"
+
+	"github.com/micro/cli"
+	"github.com/micro/go-micro/agent/input"
+	"github.com/nlopes/slack"
+)
+
+type slackInput struct {
+	debug bool
+	token string
+
+	sync.Mutex
+	running bool
+	exit    chan bool
+
+	api *slack.Client
+}
+
+func init() {
+	input.Inputs["slack"] = NewInput()
+}
+
+func (p *slackInput) Flags() []cli.Flag {
+	return []cli.Flag{
+		cli.BoolFlag{
+			Name:   "slack_debug",
+			Usage:  "Slack debug output",
+			EnvVar: "MICRO_SLACK_DEBUG",
+		},
+		cli.StringFlag{
+			Name:   "slack_token",
+			Usage:  "Slack token",
+			EnvVar: "MICRO_SLACK_TOKEN",
+		},
+	}
+}
+
+func (p *slackInput) Init(ctx *cli.Context) error {
+	debug := ctx.Bool("slack_debug")
+	token := ctx.String("slack_token")
+
+	if len(token) == 0 {
+		return errors.New("missing slack token")
+	}
+
+	p.debug = debug
+	p.token = token
+
+	return nil
+}
+
+func (p *slackInput) Stream() (input.Conn, error) {
+	p.Lock()
+	defer p.Unlock()
+
+	if !p.running {
+		return nil, errors.New("not running")
+	}
+
+	// test auth
+	auth, err := p.api.AuthTest()
+	if err != nil {
+		return nil, err
+	}
+
+	rtm := p.api.NewRTM()
+	exit := make(chan bool)
+
+	go rtm.ManageConnection()
+
+	go func() {
+		select {
+		case <-p.exit:
+			select {
+			case <-exit:
+				return
+			default:
+				close(exit)
+			}
+		case <-exit:
+		}
+
+		rtm.Disconnect()
+	}()
+
+	conn := &slackConn{
+		auth:  auth,
+		rtm:   rtm,
+		exit:  exit,
+		names: make(map[string]string),
+	}
+
+	go conn.run()
+
+	return conn, nil
+}
+
+func (p *slackInput) Start() error {
+	if len(p.token) == 0 {
+		return errors.New("missing slack token")
+	}
+
+	p.Lock()
+	defer p.Unlock()
+
+	if p.running {
+		return nil
+	}
+
+	api := slack.New(p.token, slack.OptionDebug(p.debug))
+
+	// test auth
+	_, err := api.AuthTest()
+	if err != nil {
+		return err
+	}
+
+	p.api = api
+	p.exit = make(chan bool)
+	p.running = true
+	return nil
+}
+
+func (p *slackInput) Stop() error {
+	p.Lock()
+	defer p.Unlock()
+
+	if !p.running {
+		return nil
+	}
+
+	close(p.exit)
+	p.running = false
+	return nil
+}
+
+func (p *slackInput) String() string {
+	return "slack"
+}
+
+func NewInput() input.Input {
+	return &slackInput{}
+}

agent/input/telegram/README.md

@@ -0,0 +1,18 @@
+# Telegram Messenger input for micro bot
+[Telegram](https://telegram.org) support for micro bot based on [telegram-bot-api](https://github.com/go-telegram-bot-api/telegram-bot-api).
+
+## Options
+### --telegram_token (required)
+
+Sets bot's token for interacting with API.
+
+Head over to Telegram's [API documentation](https://core.telegram.org/bots/api)
+to learn how to create bots and how the API works.
+
+### --telegram_debug
+
+Sets the debug flag to make the bot's output verbose.
+
+### --telegram_whitelist
+
+Sets a list of comma-separated nicknames (without @ symbol in the beginning) for interacting with bot. Only these users can use the bot.

agent/input/telegram/conn.go

@@ -0,0 +1,115 @@
+package telegram
+
+import (
+	"errors"
+	"strings"
+	"sync"
+
+	"github.com/forestgiant/sliceutil"
+	"github.com/micro/go-micro/agent/input"
+	"github.com/micro/go-micro/util/log"
+	"gopkg.in/telegram-bot-api.v4"
+)
+
+type telegramConn struct {
+	input *telegramInput
+
+	recv <-chan tgbotapi.Update
+	exit chan bool
+
+	syncCond *sync.Cond
+	mutex    sync.Mutex
+}
+
+func newConn(input *telegramInput) (*telegramConn, error) {
+	conn := &telegramConn{
+		input: input,
+	}
+
+	conn.syncCond = sync.NewCond(&conn.mutex)
+
+	go conn.run()
+
+	return conn, nil
+}
+
+func (tc *telegramConn) run() {
+	u := tgbotapi.NewUpdate(0)
+	u.Timeout = 60
+	updates, err := tc.input.api.GetUpdatesChan(u)
+	if err != nil {
+		return
+	}
+
+	tc.recv = updates
+	tc.syncCond.Signal()
+
+	for {
+		select {
+		case <-tc.exit:
+			return
+		}
+	}
+}
+
+func (tc *telegramConn) Close() error {
+	return nil
+}
+
+func (tc *telegramConn) Recv(event *input.Event) error {
+	if event == nil {
+		return errors.New("event cannot be nil")
+	}
+
+	for {
+		if tc.recv == nil {
+			tc.mutex.Lock()
+			tc.syncCond.Wait()
+		}
+
+		update := <-tc.recv
+
+		if update.Message == nil || (len(tc.input.whitelist) > 0 && !sliceutil.Contains(tc.input.whitelist, update.Message.From.UserName)) {
+			continue
+		}
+
+		if event.Meta == nil {
+			event.Meta = make(map[string]interface{})
+		}
+
+		event.Type = input.TextEvent
+		event.From = update.Message.From.UserName
+		event.To = tc.input.api.Self.UserName
+		event.Data = []byte(update.Message.Text)
+		event.Meta["chatId"] = update.Message.Chat.ID
+		event.Meta["chatType"] = update.Message.Chat.Type
+		event.Meta["messageId"] = update.Message.MessageID
+
+		return nil
+	}
+}
+
+func (tc *telegramConn) Send(event *input.Event) error {
+	messageText := strings.TrimSpace(string(event.Data))
+
+	chatId := event.Meta["chatId"].(int64)
+	chatType := ChatType(event.Meta["chatType"].(string))
+
+	msgConfig := tgbotapi.NewMessage(chatId, messageText)
+	msgConfig.ParseMode = tgbotapi.ModeHTML
+
+	if sliceutil.Contains([]ChatType{Group, Supergroup}, chatType) {
+		msgConfig.ReplyToMessageID = event.Meta["messageId"].(int)
+	}
+
+	_, err := tc.input.api.Send(msgConfig)
+
+	if err != nil {
+		// probably it could be because of nested HTML tags -- telegram doesn't allow nested tags
+		log.Log("[telegram][Send] error:", err)
+		msgConfig.Text = "This bot couldn't send the response (Internal error)"
+		tc.input.api.Send(msgConfig)
+	}
+
+	return nil
+}

agent/input/telegram/telegram.go

@@ -0,0 +1,101 @@
+package telegram
+
+import (
+	"errors"
+	"strings"
+	"sync"
+
+	"github.com/micro/cli"
+	"github.com/micro/go-micro/agent/input"
+	"gopkg.in/telegram-bot-api.v4"
+)
+
+type telegramInput struct {
+	sync.Mutex
+
+	debug     bool
+	token     string
+	whitelist []string
+
+	api *tgbotapi.BotAPI
+}
+
+type ChatType string
+
+const (
+	Private    ChatType = "private"
+	Group      ChatType = "group"
+	Supergroup ChatType = "supergroup"
+)
+
+func init() {
+	input.Inputs["telegram"] = &telegramInput{}
+}
+
+func (ti *telegramInput) Flags() []cli.Flag {
+	return []cli.Flag{
+		cli.BoolFlag{
+			Name:   "telegram_debug",
+			EnvVar: "MICRO_TELEGRAM_DEBUG",
+			Usage:  "Telegram debug output",
+		},
+		cli.StringFlag{
+			Name:   "telegram_token",
+			EnvVar: "MICRO_TELEGRAM_TOKEN",
+			Usage:  "Telegram token",
+		},
+		cli.StringFlag{
+			Name:   "telegram_whitelist",
+			EnvVar: "MICRO_TELEGRAM_WHITELIST",
+			Usage:  "Telegram bot's users (comma-separated values)",
+		},
+	}
+}
+
+func (ti *telegramInput) Init(ctx *cli.Context) error {
+	ti.debug = ctx.Bool("telegram_debug")
+	ti.token = ctx.String("telegram_token")
+
+	whitelist := ctx.String("telegram_whitelist")
+
+	if whitelist != "" {
+		ti.whitelist = strings.Split(whitelist, ",")
+	}
+
+	if len(ti.token) == 0 {
+		return errors.New("missing telegram token")
+	}
+
+	return nil
+}
+
+func (ti *telegramInput) Stream() (input.Conn, error) {
+	ti.Lock()
+	defer ti.Unlock()
+
+	return newConn(ti)
+}
+
+func (ti *telegramInput) Start() error {
+	ti.Lock()
+	defer ti.Unlock()
+
+	api, err := tgbotapi.NewBotAPI(ti.token)
+	if err != nil {
+		return err
+	}
+
+	ti.api = api
+
+	api.Debug = ti.debug
+
+	return nil
+}
+
+func (ti *telegramInput) Stop() error {
+	return nil
+}
+
+func (p *telegramInput) String() string {
+	return "telegram"
+}

agent/proto/bot.micro.go

@@ -0,0 +1,118 @@
+// Code generated by protoc-gen-micro. DO NOT EDIT.
+// source: github.com/micro/go-bot/proto/bot.proto
+
+/*
+Package go_micro_bot is a generated protocol buffer package.
+
+It is generated from these files:
+	github.com/micro/go-bot/proto/bot.proto
+
+It has these top-level messages:
+	HelpRequest
+	HelpResponse
+	ExecRequest
+	ExecResponse
+*/
+package go_micro_bot
+
+import proto "github.com/golang/protobuf/proto"
+import fmt "fmt"
+import math "math"
+
+import (
+	context "context"
+	client "github.com/micro/go-micro/client"
+	server "github.com/micro/go-micro/server"
+)
+
+// Reference imports to suppress errors if they are not otherwise used.
+var _ = proto.Marshal
+var _ = fmt.Errorf
+var _ = math.Inf
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the proto package it is being compiled against.
+// A compilation error at this line likely means your copy of the
+// proto package needs to be updated.
+const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package
+
+// Reference imports to suppress errors if they are not otherwise used.
+var _ context.Context
+var _ client.Option
+var _ server.Option
+
+// Client API for Command service
+
+type CommandService interface {
+	Help(ctx context.Context, in *HelpRequest, opts ...client.CallOption) (*HelpResponse, error)
+	Exec(ctx context.Context, in *ExecRequest, opts ...client.CallOption) (*ExecResponse, error)
+}
+
+type commandService struct {
+	c    client.Client
+	name string
+}
+
+func NewCommandService(name string, c client.Client) CommandService {
+	if c == nil {
+		c = client.NewClient()
+	}
+	if len(name) == 0 {
+		name = "go.micro.bot"
+	}
+	return &commandService{
+		c:    c,
+		name: name,
+	}
+}
+
+func (c *commandService) Help(ctx context.Context, in *HelpRequest, opts ...client.CallOption) (*HelpResponse, error) {
+	req := c.c.NewRequest(c.name, "Command.Help", in)
+	out := new(HelpResponse)
+	err := c.c.Call(ctx, req, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+func (c *commandService) Exec(ctx context.Context, in *ExecRequest, opts ...client.CallOption) (*ExecResponse, error) {
+	req := c.c.NewRequest(c.name, "Command.Exec", in)
+	out := new(ExecResponse)
+	err := c.c.Call(ctx, req, out, opts...)
+	if err != nil {
+		return nil, err
+	}
+	return out, nil
+}
+
+// Server API for Command service
+
+type CommandHandler interface {
+	Help(context.Context, *HelpRequest, *HelpResponse) error
+	Exec(context.Context, *ExecRequest, *ExecResponse) error
+}
+
+func RegisterCommandHandler(s server.Server, hdlr CommandHandler, opts ...server.HandlerOption) error {
+	type _command interface {
+		Help(ctx context.Context, in *HelpRequest, out *HelpResponse) error
+		Exec(ctx context.Context, in *ExecRequest, out *ExecResponse) error
+	}
+	type Command struct {
+		_command
+	}
+	h := &commandHandler{hdlr}
+	return s.Handle(s.NewHandler(&Command{h}, opts...))
+}
+
+type commandHandler struct {
+	CommandHandler
+}
+
+func (h *commandHandler) Help(ctx context.Context, in *HelpRequest, out *HelpResponse) error {
+	return h.CommandHandler.Help(ctx, in, out)
+}
+
+func (h *commandHandler) Exec(ctx context.Context, in *ExecRequest, out *ExecResponse) error {
+	return h.CommandHandler.Exec(ctx, in, out)
+}

agent/proto/bot.pb.go

@@ -0,0 +1,210 @@
+// Code generated by protoc-gen-go. DO NOT EDIT.
+// source: github.com/micro/go-bot/proto/bot.proto
+
+package go_micro_bot
+
+import proto "github.com/golang/protobuf/proto"
+import fmt "fmt"
+import math "math"
+
+// Reference imports to suppress errors if they are not otherwise used.
+var _ = proto.Marshal
+var _ = fmt.Errorf
+var _ = math.Inf
+
+// This is a compile-time assertion to ensure that this generated file
+// is compatible with the proto package it is being compiled against.
+// A compilation error at this line likely means your copy of the
+// proto package needs to be updated.
+const _ = proto.ProtoPackageIsVersion2 // please upgrade the proto package
+
+type HelpRequest struct {
+	XXX_NoUnkeyedLiteral struct{} `json:"-"`
+	XXX_unrecognized     []byte   `json:"-"`
+	XXX_sizecache        int32    `json:"-"`
+}
+
+func (m *HelpRequest) Reset()         { *m = HelpRequest{} }
+func (m *HelpRequest) String() string { return proto.CompactTextString(m) }
+func (*HelpRequest) ProtoMessage()    {}
+func (*HelpRequest) Descriptor() ([]byte, []int) {
+	return fileDescriptor_bot_654832eab83ed4b5, []int{0}
+}
+func (m *HelpRequest) XXX_Unmarshal(b []byte) error {
+	return xxx_messageInfo_HelpRequest.Unmarshal(m, b)
+}
+func (m *HelpRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	return xxx_messageInfo_HelpRequest.Marshal(b, m, deterministic)
+}
+func (dst *HelpRequest) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_HelpRequest.Merge(dst, src)
+}
+func (m *HelpRequest) XXX_Size() int {
+	return xxx_messageInfo_HelpRequest.Size(m)
+}
+func (m *HelpRequest) XXX_DiscardUnknown() {
+	xxx_messageInfo_HelpRequest.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_HelpRequest proto.InternalMessageInfo
+
+type HelpResponse struct {
+	Usage                string   `protobuf:"bytes,1,opt,name=usage,proto3" json:"usage,omitempty"`
+	Description          string   `protobuf:"bytes,2,opt,name=description,proto3" json:"description,omitempty"`
+	XXX_NoUnkeyedLiteral struct{} `json:"-"`
+	XXX_unrecognized     []byte   `json:"-"`
+	XXX_sizecache        int32    `json:"-"`
+}
+
+func (m *HelpResponse) Reset()         { *m = HelpResponse{} }
+func (m *HelpResponse) String() string { return proto.CompactTextString(m) }
+func (*HelpResponse) ProtoMessage()    {}
+func (*HelpResponse) Descriptor() ([]byte, []int) {
+	return fileDescriptor_bot_654832eab83ed4b5, []int{1}
+}
+func (m *HelpResponse) XXX_Unmarshal(b []byte) error {
+	return xxx_messageInfo_HelpResponse.Unmarshal(m, b)
+}
+func (m *HelpResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	return xxx_messageInfo_HelpResponse.Marshal(b, m, deterministic)
+}
+func (dst *HelpResponse) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_HelpResponse.Merge(dst, src)
+}
+func (m *HelpResponse) XXX_Size() int {
+	return xxx_messageInfo_HelpResponse.Size(m)
+}
+func (m *HelpResponse) XXX_DiscardUnknown() {
+	xxx_messageInfo_HelpResponse.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_HelpResponse proto.InternalMessageInfo
+
+func (m *HelpResponse) GetUsage() string {
+	if m != nil {
+		return m.Usage
+	}
+	return ""
+}
+
+func (m *HelpResponse) GetDescription() string {
+	if m != nil {
+		return m.Description
+	}
+	return ""
+}
+
+type ExecRequest struct {
+	Args                 []string `protobuf:"bytes,1,rep,name=args,proto3" json:"args,omitempty"`
+	XXX_NoUnkeyedLiteral struct{} `json:"-"`
+	XXX_unrecognized     []byte   `json:"-"`
+	XXX_sizecache        int32    `json:"-"`
+}
+
+func (m *ExecRequest) Reset()         { *m = ExecRequest{} }
+func (m *ExecRequest) String() string { return proto.CompactTextString(m) }
+func (*ExecRequest) ProtoMessage()    {}
+func (*ExecRequest) Descriptor() ([]byte, []int) {
+	return fileDescriptor_bot_654832eab83ed4b5, []int{2}
+}
+func (m *ExecRequest) XXX_Unmarshal(b []byte) error {
+	return xxx_messageInfo_ExecRequest.Unmarshal(m, b)
+}
+func (m *ExecRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	return xxx_messageInfo_ExecRequest.Marshal(b, m, deterministic)
+}
+func (dst *ExecRequest) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_ExecRequest.Merge(dst, src)
+}
+func (m *ExecRequest) XXX_Size() int {
+	return xxx_messageInfo_ExecRequest.Size(m)
+}
+func (m *ExecRequest) XXX_DiscardUnknown() {
+	xxx_messageInfo_ExecRequest.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_ExecRequest proto.InternalMessageInfo
+
+func (m *ExecRequest) GetArgs() []string {
+	if m != nil {
+		return m.Args
+	}
+	return nil
+}
+
+type ExecResponse struct {
+	Result               []byte   `protobuf:"bytes,1,opt,name=result,proto3" json:"result,omitempty"`
+	Error                string   `protobuf:"bytes,2,opt,name=error,proto3" json:"error,omitempty"`
+	XXX_NoUnkeyedLiteral struct{} `json:"-"`
+	XXX_unrecognized     []byte   `json:"-"`
+	XXX_sizecache        int32    `json:"-"`
+}
+
+func (m *ExecResponse) Reset()         { *m = ExecResponse{} }
+func (m *ExecResponse) String() string { return proto.CompactTextString(m) }
+func (*ExecResponse) ProtoMessage()    {}
+func (*ExecResponse) Descriptor() ([]byte, []int) {
+	return fileDescriptor_bot_654832eab83ed4b5, []int{3}
+}
+func (m *ExecResponse) XXX_Unmarshal(b []byte) error {
+	return xxx_messageInfo_ExecResponse.Unmarshal(m, b)
+}
+func (m *ExecResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) {
+	return xxx_messageInfo_ExecResponse.Marshal(b, m, deterministic)
+}
+func (dst *ExecResponse) XXX_Merge(src proto.Message) {
+	xxx_messageInfo_ExecResponse.Merge(dst, src)
+}
+func (m *ExecResponse) XXX_Size() int {
+	return xxx_messageInfo_ExecResponse.Size(m)
+}
+func (m *ExecResponse) XXX_DiscardUnknown() {
+	xxx_messageInfo_ExecResponse.DiscardUnknown(m)
+}
+
+var xxx_messageInfo_ExecResponse proto.InternalMessageInfo
+
+func (m *ExecResponse) GetResult() []byte {
+	if m != nil {
+		return m.Result
+	}
+	return nil
+}
+
+func (m *ExecResponse) GetError() string {
+	if m != nil {
+		return m.Error
+	}
+	return ""
+}
+
+func init() {
+	proto.RegisterType((*HelpRequest)(nil), "go.micro.bot.HelpRequest")
+	proto.RegisterType((*HelpResponse)(nil), "go.micro.bot.HelpResponse")
+	proto.RegisterType((*ExecRequest)(nil), "go.micro.bot.ExecRequest")
+	proto.RegisterType((*ExecResponse)(nil), "go.micro.bot.ExecResponse")
+}
+
+func init() {
+	proto.RegisterFile("github.com/micro/go-bot/proto/bot.proto", fileDescriptor_bot_654832eab83ed4b5)
+}
+
+var fileDescriptor_bot_654832eab83ed4b5 = []byte{
+	// 246 bytes of a gzipped FileDescriptorProto
+	0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x6c, 0x90, 0x41, 0x4b, 0xc4, 0x30,
+	0x10, 0x85, 0xb7, 0xba, 0xae, 0xec, 0xb4, 0x5e, 0x82, 0x48, 0xdd, 0x53, 0xcd, 0xc5, 0xbd, 0x98,
+	0x82, 0x5e, 0x05, 0x0f, 0xa2, 0x78, 0xee, 0x3f, 0x68, 0xba, 0x43, 0x2c, 0x6c, 0x3b, 0x35, 0x99,
+	0x82, 0xff, 0xc1, 0x3f, 0x2d, 0x4d, 0x72, 0x08, 0xcb, 0xde, 0xe6, 0x65, 0x86, 0xf7, 0xbe, 0x17,
+	0x78, 0x34, 0x3d, 0x7f, 0xcf, 0x5a, 0x75, 0x34, 0xd4, 0x43, 0xdf, 0x59, 0xaa, 0x0d, 0x3d, 0x69,
+	0xe2, 0x7a, 0xb2, 0xc4, 0x54, 0x6b, 0x62, 0xe5, 0x27, 0x51, 0x18, 0x52, 0xfe, 0x40, 0x69, 0x62,
+	0x79, 0x03, 0xf9, 0x17, 0x1e, 0xa7, 0x06, 0x7f, 0x66, 0x74, 0x2c, 0x3f, 0xa1, 0x08, 0xd2, 0x4d,
+	0x34, 0x3a, 0x14, 0xb7, 0x70, 0x35, 0xbb, 0xd6, 0x60, 0x99, 0x55, 0xd9, 0x7e, 0xdb, 0x04, 0x21,
+	0x2a, 0xc8, 0x0f, 0xe8, 0x3a, 0xdb, 0x4f, 0xdc, 0xd3, 0x58, 0x5e, 0xf8, 0x5d, 0xfa, 0x24, 0x1f,
+	0x20, 0xff, 0xf8, 0xc5, 0x2e, 0xda, 0x0a, 0x01, 0xeb, 0xd6, 0x1a, 0x57, 0x66, 0xd5, 0xe5, 0x7e,
+	0xdb, 0xf8, 0x59, 0xbe, 0x42, 0x11, 0x4e, 0x62, 0xd4, 0x1d, 0x6c, 0x2c, 0xba, 0xf9, 0xc8, 0x3e,
+	0xab, 0x68, 0xa2, 0x5a, 0x10, 0xd0, 0x5a, 0xb2, 0x31, 0x26, 0x88, 0xe7, 0xbf, 0x0c, 0xae, 0xdf,
+	0x69, 0x18, 0xda, 0xf1, 0x20, 0xde, 0x60, 0xbd, 0x40, 0x8b, 0x7b, 0x95, 0x56, 0x53, 0x49, 0xaf,
+	0xdd, 0xee, 0xdc, 0x2a, 0x04, 0xcb, 0xd5, 0x62, 0xb0, 0xa0, 0x9c, 0x1a, 0x24, 0x0d, 0x4e, 0x0d,
+	0x52, 0x72, 0xb9, 0xd2, 0x1b, 0xff, 0xb5, 0x2f, 0xff, 0x01, 0x00, 0x00, 0xff, 0xff, 0xcb, 0x77,
+	0xdf, 0x28, 0x85, 0x01, 0x00, 0x00,
+}

agent/proto/bot.proto

@@ -0,0 +1,25 @@
+syntax = "proto3";
+
+package go.micro.bot;
+
+service Command {
+	rpc Help(HelpRequest) returns (HelpResponse) {};
+	rpc Exec(ExecRequest) returns (ExecResponse) {};
+}
+
+message HelpRequest {
+}
+
+message HelpResponse {
+	string usage = 1;
+	string description = 2;
+}
+
+message ExecRequest {
+	repeated string args = 1;
+}
+
+message ExecResponse {
+	bytes result = 1;
+	string error = 2;
+}

broker/http_broker.go

@@ -22,10 +22,10 @@ import (
 	"github.com/micro/go-micro/codec/json"
 	merr "github.com/micro/go-micro/errors"
 	"github.com/micro/go-micro/registry"
+	maddr "github.com/micro/go-micro/util/addr"
+	mnet "github.com/micro/go-micro/util/net"
+	mls "github.com/micro/go-micro/util/tls"
 	"github.com/micro/go-rcache"
-	maddr "github.com/micro/util/go/lib/addr"
-	mnet "github.com/micro/util/go/lib/net"
-	mls "github.com/micro/util/go/lib/tls"
 	"golang.org/x/net/http2"
 )
 

broker/http_broker_test.go

@@ -7,8 +7,8 @@ import (
 
 	glog "github.com/go-log/log"
 	"github.com/google/uuid"
-	"github.com/micro/go-log"
 	"github.com/micro/go-micro/registry/memory"
+	"github.com/micro/go-micro/util/log"
 )
 
 func newTestRegistry() *memory.Registry {

cmd/cmd.go

@@ -10,9 +10,9 @@ import (
 	"time"
 
 	"github.com/micro/cli"
-	"github.com/micro/go-log"
 	"github.com/micro/go-micro/client"
 	"github.com/micro/go-micro/server"
+	"github.com/micro/go-micro/util/log"
 
 	// brokers
 	"github.com/micro/go-micro/broker"

codec/text/text.go

@@ -0,0 +1,75 @@
+// Package text reads any text/* content-type
+package text
+
+import (
+	"fmt"
+	"io"
+	"io/ioutil"
+
+	"github.com/micro/go-micro/codec"
+)
+
+type Codec struct {
+	Conn io.ReadWriteCloser
+}
+
+// Frame gives us the ability to define raw data to send over the pipes
+type Frame struct {
+	Data []byte
+}
+
+func (c *Codec) ReadHeader(m *codec.Message, t codec.MessageType) error {
+	return nil
+}
+
+func (c *Codec) ReadBody(b interface{}) error {
+	// read bytes
+	buf, err := ioutil.ReadAll(c.Conn)
+	if err != nil {
+		return err
+	}
+
+	switch b.(type) {
+	case *[]byte:
+		v := b.(*[]byte)
+		*v = buf
+	case *Frame:
+		v := b.(*Frame)
+		v.Data = buf
+	default:
+		return fmt.Errorf("failed to read body: %v is not type of *[]byte", b)
+	}
+
+	return nil
+}
+
+func (c *Codec) Write(m *codec.Message, b interface{}) error {
+	var v []byte
+	switch b.(type) {
+	case *Frame:
+		v = b.(*Frame).Data
+	case *[]byte:
+		ve := b.(*[]byte)
+		v = *ve
+	case []byte:
+		v = b.([]byte)
+	default:
+		return fmt.Errorf("failed to write: %v is not type of *[]byte or []byte", b)
+	}
+	_, err := c.Conn.Write(v)
+	return err
+}
+
+func (c *Codec) Close() error {
+	return c.Conn.Close()
+}
+
+func (c *Codec) String() string {
+	return "bytes"
+}
+
+func NewCodec(c io.ReadWriteCloser) codec.Codec {
+	return &Codec{
+		Conn: c,
+	}
+}

config/README.md

@@ -0,0 +1,29 @@
+# Config [![GoDoc](https://godoc.org/github.com/micro/go-micro/config?status.svg)](https://godoc.org/github.com/micro/go-micro/config)
+
+Go Config is a pluggable dynamic config library.
+
+Most config in applications are statically configured or include complex logic to load from multiple sources. 
+Go Config makes this easy, pluggable and mergeable. You'll never have to deal with config in the same way again.
+
+## Features
+
+- **Dynamic Loading** - Load configuration from multiple source as and when needed. Go Config manages watching config sources 
+in the background and automatically merges and updates an in memory view. 
+
+- **Pluggable Sources** - Choose from any number of sources to load and merge config. The backend source is abstracted away into 
+a standard format consumed internally and decoded via encoders. Sources can be env vars, flags, file, etcd, k8s configmap, etc.
+
+- **Mergeable Config** - If you specify multiple sources of config, regardless of format, they will be merged and presented in 
+a single view. This massively simplifies priority order loading and changes based on environment.
+
+- **Observe Changes** - Optionally watch the config for changes to specific values. Hot reload your app using Go Config's watcher. 
+You don't have to handle ad-hoc hup reloading or whatever else, just keep reading the config and watch for changes if you need 
+to be notified.
+
+- **Sane Defaults** - In case config loads badly or is completely wiped away for some unknown reason, you can specify fallback 
+values when accessing any config values directly. This ensures you'll always be reading some sane default in the event of a problem.
+
+## Getting Started
+
+For detailed information or architecture, installation and general usage see the [docs](https://micro.mu/docs/go-config.html)
+

config/config.go

@@ -0,0 +1,94 @@
+// Package config is an interface for dynamic configuration.
+package config
+
+import (
+	"context"
+
+	"github.com/micro/go-micro/config/loader"
+	"github.com/micro/go-micro/config/reader"
+	"github.com/micro/go-micro/config/source"
+	"github.com/micro/go-micro/config/source/file"
+)
+
+// Config is an interface abstraction for dynamic configuration
+type Config interface {
+	// provide the reader.Values interface
+	reader.Values
+	// Stop the config loader/watcher
+	Close() error
+	// Load config sources
+	Load(source ...source.Source) error
+	// Force a source changeset sync
+	Sync() error
+	// Watch a value for changes
+	Watch(path ...string) (Watcher, error)
+}
+
+// Watcher is the config watcher
+type Watcher interface {
+	Next() (reader.Value, error)
+	Stop() error
+}
+
+type Options struct {
+	Loader loader.Loader
+	Reader reader.Reader
+	Source []source.Source
+
+	// for alternative data
+	Context context.Context
+}
+
+type Option func(o *Options)
+
+var (
+	// Default Config Manager
+	DefaultConfig = NewConfig()
+)
+
+// NewConfig returns new config
+func NewConfig(opts ...Option) Config {
+	return newConfig(opts...)
+}
+
+// Return config as raw json
+func Bytes() []byte {
+	return DefaultConfig.Bytes()
+}
+
+// Return config as a map
+func Map() map[string]interface{} {
+	return DefaultConfig.Map()
+}
+
+// Scan values to a go type
+func Scan(v interface{}) error {
+	return DefaultConfig.Scan(v)
+}
+
+// Force a source changeset sync
+func Sync() error {
+	return DefaultConfig.Sync()
+}
+
+// Get a value from the config
+func Get(path ...string) reader.Value {
+	return DefaultConfig.Get(path...)
+}
+
+// Load config sources
+func Load(source ...source.Source) error {
+	return DefaultConfig.Load(source...)
+}
+
+// Watch a value for changes
+func Watch(path ...string) (Watcher, error) {
+	return DefaultConfig.Watch(path...)
+}
+
+// LoadFile is short hand for creating a file source and loading it
+func LoadFile(path string) error {
+	return Load(file.NewSource(
+		file.WithPath(path),
+	))
+}

config/default.go

@@ -0,0 +1,253 @@
+package config
+
+import (
+	"bytes"
+	"sync"
+	"time"
+
+	"github.com/micro/go-micro/config/loader"
+	"github.com/micro/go-micro/config/loader/memory"
+	"github.com/micro/go-micro/config/reader"
+	"github.com/micro/go-micro/config/reader/json"
+	"github.com/micro/go-micro/config/source"
+)
+
+type config struct {
+	exit chan bool
+	opts Options
+
+	sync.RWMutex
+	// the current snapshot
+	snap *loader.Snapshot
+	// the current values
+	vals reader.Values
+}
+
+type watcher struct {
+	lw    loader.Watcher
+	rd    reader.Reader
+	path  []string
+	value reader.Value
+}
+
+func newConfig(opts ...Option) Config {
+	options := Options{
+		Loader: memory.NewLoader(),
+		Reader: json.NewReader(),
+	}
+
+	for _, o := range opts {
+		o(&options)
+	}
+
+	options.Loader.Load(options.Source...)
+	snap, _ := options.Loader.Snapshot()
+	vals, _ := options.Reader.Values(snap.ChangeSet)
+
+	c := &config{
+		exit: make(chan bool),
+		opts: options,
+		snap: snap,
+		vals: vals,
+	}
+
+	go c.run()
+
+	return c
+}
+
+func (c *config) run() {
+	watch := func(w loader.Watcher) error {
+		for {
+			// get changeset
+			snap, err := w.Next()
+			if err != nil {
+				return err
+			}
+
+			c.Lock()
+
+			// save
+			c.snap = snap
+
+			// set values
+			c.vals, _ = c.opts.Reader.Values(snap.ChangeSet)
+
+			c.Unlock()
+		}
+	}
+
+	for {
+		w, err := c.opts.Loader.Watch()
+		if err != nil {
+			time.Sleep(time.Second)
+			continue
+		}
+
+		done := make(chan bool)
+
+		// the stop watch func
+		go func() {
+			select {
+			case <-done:
+			case <-c.exit:
+			}
+			w.Stop()
+		}()
+
+		// block watch
+		if err := watch(w); err != nil {
+			// do something better
+			time.Sleep(time.Second)
+		}
+
+		// close done chan
+		close(done)
+
+		// if the config is closed exit
+		select {
+		case <-c.exit:
+			return
+		default:
+		}
+	}
+}
+
+func (c *config) Map() map[string]interface{} {
+	c.RLock()
+	defer c.RUnlock()
+	return c.vals.Map()
+}
+
+func (c *config) Scan(v interface{}) error {
+	c.RLock()
+	defer c.RUnlock()
+	return c.vals.Scan(v)
+}
+
+// sync loads all the sources, calls the parser and updates the config
+func (c *config) Sync() error {
+	if err := c.opts.Loader.Sync(); err != nil {
+		return err
+	}
+
+	snap, err := c.opts.Loader.Snapshot()
+	if err != nil {
+		return err
+	}
+
+	c.Lock()
+	defer c.Unlock()
+
+	c.snap = snap
+	vals, err := c.opts.Reader.Values(snap.ChangeSet)
+	if err != nil {
+		return err
+	}
+	c.vals = vals
+
+	return nil
+}
+
+func (c *config) Close() error {
+	select {
+	case <-c.exit:
+		return nil
+	default:
+		close(c.exit)
+	}
+	return nil
+}
+
+func (c *config) Get(path ...string) reader.Value {
+	c.RLock()
+	defer c.RUnlock()
+
+	// did sync actually work?
+	if c.vals != nil {
+		return c.vals.Get(path...)
+	}
+
+	// no value
+	return newValue()
+}
+
+func (c *config) Bytes() []byte {
+	c.RLock()
+	defer c.RUnlock()
+
+	if c.vals == nil {
+		return []byte{}
+	}
+
+	return c.vals.Bytes()
+}
+
+func (c *config) Load(sources ...source.Source) error {
+	if err := c.opts.Loader.Load(sources...); err != nil {
+		return err
+	}
+
+	snap, err := c.opts.Loader.Snapshot()
+	if err != nil {
+		return err
+	}
+
+	c.Lock()
+	defer c.Unlock()
+
+	c.snap = snap
+	vals, err := c.opts.Reader.Values(snap.ChangeSet)
+	if err != nil {
+		return err
+	}
+	c.vals = vals
+
+	return nil
+}
+
+func (c *config) Watch(path ...string) (Watcher, error) {
+	value := c.Get(path...)
+
+	w, err := c.opts.Loader.Watch(path...)
+	if err != nil {
+		return nil, err
+	}
+
+	return &watcher{
+		lw:    w,
+		rd:    c.opts.Reader,
+		path:  path,
+		value: value,
+	}, nil
+}
+
+func (c *config) String() string {
+	return "config"
+}
+
+func (w *watcher) Next() (reader.Value, error) {
+	for {
+		s, err := w.lw.Next()
+		if err != nil {
+			return nil, err
+		}
+
+		// only process changes
+		if bytes.Equal(w.value.Bytes(), s.ChangeSet.Data) {
+			continue
+		}
+
+		v, err := w.rd.Values(s.ChangeSet)
+		if err != nil {
+			return nil, err
+		}
+
+		w.value = v.Get()
+		return w.value, nil
+	}
+}
+
+func (w *watcher) Stop() error {
+	return w.lw.Stop()
+}

config/default_test.go

@@ -0,0 +1,101 @@
+package config
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/micro/go-micro/config/source/file"
+)
+
+func createFileForTest(t *testing.T) *os.File {
+	data := []byte(`{"foo": "bar"}`)
+	path := filepath.Join(os.TempDir(), fmt.Sprintf("file.%d", time.Now().UnixNano()))
+	fh, err := os.Create(path)
+	if err != nil {
+		t.Error(err)
+	}
+	_, err = fh.Write(data)
+	if err != nil {
+		t.Error(err)
+	}
+
+	return fh
+}
+
+func TestLoadWithGoodFile(t *testing.T) {
+	fh := createFileForTest(t)
+	path := fh.Name()
+	defer func() {
+		fh.Close()
+		os.Remove(path)
+	}()
+
+	// Create new config
+	conf := NewConfig()
+	// Load file source
+	if err := conf.Load(file.NewSource(
+		file.WithPath(path),
+	)); err != nil {
+		t.Fatalf("Expected no error but got %v", err)
+	}
+}
+
+func TestLoadWithInvalidFile(t *testing.T) {
+	fh := createFileForTest(t)
+	path := fh.Name()
+	defer func() {
+		fh.Close()
+		os.Remove(path)
+	}()
+
+	// Create new config
+	conf := NewConfig()
+	// Load file source
+	err := conf.Load(file.NewSource(
+		file.WithPath(path),
+		file.WithPath("/i/do/not/exists.json"),
+	))
+
+	if err == nil {
+		t.Fatal("Expected error but none !")
+	}
+	if !strings.Contains(fmt.Sprintf("%v", err), "/i/do/not/exists.json") {
+		t.Fatalf("Expected error to contain the unexisting file but got %v", err)
+	}
+}
+
+func TestConsul(t *testing.T) {
+	/*consulSource := consul.NewSource(
+		// optionally specify consul address; default to localhost:8500
+		consul.WithAddress("131.150.38.111:8500"),
+		// optionally specify prefix; defaults to /micro/config
+		consul.WithPrefix("/project"),
+		// optionally strip the provided prefix from the keys, defaults to false
+		consul.StripPrefix(true),
+		consul.WithDatacenter("dc1"),
+		consul.WithToken("xxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx"),
+	)
+
+	// Create new config
+	conf := NewConfig()
+
+	// Load file source
+	err := conf.Load(consulSource)
+	if err != nil {
+		t.Error(err)
+		return
+	}
+
+	m := conf.Map()
+	t.Log("m: ", m)
+
+	v := conf.Get("project", "dc111", "port")
+
+	t.Log("v: ", v.Int(13))*/
+
+	t.Log("OK")
+}

config/encoder/encoder.go

@@ -0,0 +1,8 @@
+// Package encoder handles source encoding formats
+package encoder
+
+type Encoder interface {
+	Encode(interface{}) ([]byte, error)
+	Decode([]byte, interface{}) error
+	String() string
+}

config/encoder/hcl/hcl.go

@@ -0,0 +1,26 @@
+package hcl
+
+import (
+	"encoding/json"
+
+	"github.com/hashicorp/hcl"
+	"github.com/micro/go-micro/config/encoder"
+)
+
+type hclEncoder struct{}
+
+func (h hclEncoder) Encode(v interface{}) ([]byte, error) {
+	return json.Marshal(v)
+}
+
+func (h hclEncoder) Decode(d []byte, v interface{}) error {
+	return hcl.Unmarshal(d, v)
+}
+
+func (h hclEncoder) String() string {
+	return "hcl"
+}
+
+func NewEncoder() encoder.Encoder {
+	return hclEncoder{}
+}

config/encoder/json/json.go

@@ -0,0 +1,25 @@
+package json
+
+import (
+	"encoding/json"
+
+	"github.com/micro/go-micro/config/encoder"
+)
+
+type jsonEncoder struct{}
+
+func (j jsonEncoder) Encode(v interface{}) ([]byte, error) {
+	return json.Marshal(v)
+}
+
+func (j jsonEncoder) Decode(d []byte, v interface{}) error {
+	return json.Unmarshal(d, v)
+}
+
+func (j jsonEncoder) String() string {
+	return "json"
+}
+
+func NewEncoder() encoder.Encoder {
+	return jsonEncoder{}
+}

config/encoder/toml/toml.go

@@ -0,0 +1,32 @@
+package toml
+
+import (
+	"bytes"
+
+	"github.com/BurntSushi/toml"
+	"github.com/micro/go-micro/config/encoder"
+)
+
+type tomlEncoder struct{}
+
+func (t tomlEncoder) Encode(v interface{}) ([]byte, error) {
+	b := bytes.NewBuffer(nil)
+	defer b.Reset()
+	err := toml.NewEncoder(b).Encode(v)
+	if err != nil {
+		return nil, err
+	}
+	return b.Bytes(), nil
+}
+
+func (t tomlEncoder) Decode(d []byte, v interface{}) error {
+	return toml.Unmarshal(d, v)
+}
+
+func (t tomlEncoder) String() string {
+	return "toml"
+}
+
+func NewEncoder() encoder.Encoder {
+	return tomlEncoder{}
+}

config/encoder/xml/xml.go

@@ -0,0 +1,25 @@
+package xml
+
+import (
+	"encoding/xml"
+
+	"github.com/micro/go-micro/config/encoder"
+)
+
+type xmlEncoder struct{}
+
+func (x xmlEncoder) Encode(v interface{}) ([]byte, error) {
+	return xml.Marshal(v)
+}
+
+func (x xmlEncoder) Decode(d []byte, v interface{}) error {
+	return xml.Unmarshal(d, v)
+}
+
+func (x xmlEncoder) String() string {
+	return "xml"
+}
+
+func NewEncoder() encoder.Encoder {
+	return xmlEncoder{}
+}

config/encoder/yaml/yaml.go

@@ -0,0 +1,24 @@
+package yaml
+
+import (
+	"github.com/ghodss/yaml"
+	"github.com/micro/go-micro/config/encoder"
+)
+
+type yamlEncoder struct{}
+
+func (y yamlEncoder) Encode(v interface{}) ([]byte, error) {
+	return yaml.Marshal(v)
+}
+
+func (y yamlEncoder) Decode(d []byte, v interface{}) error {
+	return yaml.Unmarshal(d, v)
+}
+
+func (y yamlEncoder) String() string {
+	return "yaml"
+}
+
+func NewEncoder() encoder.Encoder {
+	return yamlEncoder{}
+}

config/issue18_test.go

@@ -0,0 +1,60 @@
+package config
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+
+	"github.com/micro/go-micro/config/source/env"
+	"github.com/micro/go-micro/config/source/file"
+)
+
+func createFileForIssue18(t *testing.T, content string) *os.File {
+	data := []byte(content)
+	path := filepath.Join(os.TempDir(), fmt.Sprintf("file.%d", time.Now().UnixNano()))
+	fh, err := os.Create(path)
+	if err != nil {
+		t.Error(err)
+	}
+	_, err = fh.Write(data)
+	if err != nil {
+		t.Error(err)
+	}
+
+	return fh
+}
+
+func TestIssue18(t *testing.T) {
+	fh := createFileForIssue18(t, `{
+  "amqp": {
+    "host": "rabbit.platform",
+    "port": 80
+  },
+  "handler": {
+    "exchange": "springCloudBus"
+  }
+}`)
+	path := fh.Name()
+	defer func() {
+		fh.Close()
+		os.Remove(path)
+	}()
+	os.Setenv("AMQP_HOST", "rabbit.testing.com")
+
+	conf := NewConfig()
+	conf.Load(
+		file.NewSource(
+			file.WithPath(path),
+		),
+		env.NewSource(),
+	)
+
+	actualHost := conf.Get("amqp", "host").String("backup")
+	if actualHost != "rabbit.testing.com" {
+		t.Fatalf("Expected %v but got %v",
+			"rabbit.testing.com",
+			actualHost)
+	}
+}

config/loader/loader.go

@@ -0,0 +1,63 @@
+// package loader manages loading from multiple sources
+package loader
+
+import (
+	"context"
+
+	"github.com/micro/go-micro/config/reader"
+	"github.com/micro/go-micro/config/source"
+)
+
+// Loader manages loading sources
+type Loader interface {
+	// Stop the loader
+	Close() error
+	// Load the sources
+	Load(...source.Source) error
+	// A Snapshot of loaded config
+	Snapshot() (*Snapshot, error)
+	// Force sync of sources
+	Sync() error
+	// Watch for changes
+	Watch(...string) (Watcher, error)
+	// Name of loader
+	String() string
+}
+
+// Watcher lets you watch sources and returns a merged ChangeSet
+type Watcher interface {
+	// First call to next may return the current Snapshot
+	// If you are watching a path then only the data from
+	// that path is returned.
+	Next() (*Snapshot, error)
+	// Stop watching for changes
+	Stop() error
+}
+
+// Snapshot is a merged ChangeSet
+type Snapshot struct {
+	// The merged ChangeSet
+	ChangeSet *source.ChangeSet
+	// Deterministic and comparable version of the snapshot
+	Version string
+}
+
+type Options struct {
+	Reader reader.Reader
+	Source []source.Source
+
+	// for alternative data
+	Context context.Context
+}
+
+type Option func(o *Options)
+
+// Copy snapshot
+func Copy(s *Snapshot) *Snapshot {
+	cs := *(s.ChangeSet)
+
+	return &Snapshot{
+		ChangeSet: &cs,
+		Version:   s.Version,
+	}
+}

config/loader/memory/memory.go

@@ -0,0 +1,415 @@
+package memory
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"strings"
+	"sync"
+	"time"
+
+	"github.com/micro/go-micro/config/loader"
+	"github.com/micro/go-micro/config/reader"
+	"github.com/micro/go-micro/config/reader/json"
+	"github.com/micro/go-micro/config/source"
+)
+
+type memory struct {
+	exit chan bool
+	opts loader.Options
+
+	sync.RWMutex
+	// the current snapshot
+	snap *loader.Snapshot
+	// the current values
+	vals reader.Values
+	// all the sets
+	sets []*source.ChangeSet
+	// all the sources
+	sources []source.Source
+
+	idx      int
+	watchers map[int]*watcher
+}
+
+type watcher struct {
+	exit    chan bool
+	path    []string
+	value   reader.Value
+	reader  reader.Reader
+	updates chan reader.Value
+}
+
+func (m *memory) watch(idx int, s source.Source) {
+	m.Lock()
+	m.sets = append(m.sets, &source.ChangeSet{Source: s.String()})
+	m.Unlock()
+
+	// watches a source for changes
+	watch := func(idx int, s source.Watcher) error {
+		for {
+			// get changeset
+			cs, err := s.Next()
+			if err != nil {
+				return err
+			}
+
+			m.Lock()
+
+			// save
+			m.sets[idx] = cs
+
+			// merge sets
+			set, err := m.opts.Reader.Merge(m.sets...)
+			if err != nil {
+				m.Unlock()
+				return err
+			}
+
+			// set values
+			m.vals, _ = m.opts.Reader.Values(set)
+			m.snap = &loader.Snapshot{
+				ChangeSet: set,
+				Version:   fmt.Sprintf("%d", time.Now().Unix()),
+			}
+			m.Unlock()
+
+			// send watch updates
+			m.update()
+		}
+	}
+
+	for {
+		// watch the source
+		w, err := s.Watch()
+		if err != nil {
+			time.Sleep(time.Second)
+			continue
+		}
+
+		done := make(chan bool)
+
+		// the stop watch func
+		go func() {
+			select {
+			case <-done:
+			case <-m.exit:
+			}
+			w.Stop()
+		}()
+
+		// block watch
+		if err := watch(idx, w); err != nil {
+			// do something better
+			time.Sleep(time.Second)
+		}
+
+		// close done chan
+		close(done)
+
+		// if the config is closed exit
+		select {
+		case <-m.exit:
+			return
+		default:
+		}
+	}
+}
+
+func (m *memory) loaded() bool {
+	var loaded bool
+	m.RLock()
+	if m.vals != nil {
+		loaded = true
+	}
+	m.RUnlock()
+	return loaded
+}
+
+// reload reads the sets and creates new values
+func (m *memory) reload() error {
+	m.Lock()
+
+	// merge sets
+	set, err := m.opts.Reader.Merge(m.sets...)
+	if err != nil {
+		m.Unlock()
+		return err
+	}
+
+	// set values
+	m.vals, _ = m.opts.Reader.Values(set)
+	m.snap = &loader.Snapshot{
+		ChangeSet: set,
+		Version:   fmt.Sprintf("%d", time.Now().Unix()),
+	}
+
+	m.Unlock()
+
+	// update watchers
+	m.update()
+
+	return nil
+}
+
+func (m *memory) update() {
+	var watchers []*watcher
+
+	m.RLock()
+	for _, w := range m.watchers {
+		watchers = append(watchers, w)
+	}
+	m.RUnlock()
+
+	for _, w := range watchers {
+		select {
+		case w.updates <- m.vals.Get(w.path...):
+		default:
+		}
+	}
+}
+
+// Snapshot returns a snapshot of the current loaded config
+func (m *memory) Snapshot() (*loader.Snapshot, error) {
+	if m.loaded() {
+		m.RLock()
+		snap := loader.Copy(m.snap)
+		m.RUnlock()
+		return snap, nil
+	}
+
+	// not loaded, sync
+	if err := m.Sync(); err != nil {
+		return nil, err
+	}
+
+	// make copy
+	m.RLock()
+	snap := loader.Copy(m.snap)
+	m.RUnlock()
+
+	return snap, nil
+}
+
+// Sync loads all the sources, calls the parser and updates the config
+func (m *memory) Sync() error {
+	var sets []*source.ChangeSet
+
+	m.Lock()
+
+	// read the source
+	var gerr []string
+
+	for _, source := range m.sources {
+		ch, err := source.Read()
+		if err != nil {
+			gerr = append(gerr, err.Error())
+			continue
+		}
+		sets = append(sets, ch)
+	}
+
+	// merge sets
+	set, err := m.opts.Reader.Merge(sets...)
+	if err != nil {
+		m.Unlock()
+		return err
+	}
+
+	// set values
+	vals, err := m.opts.Reader.Values(set)
+	if err != nil {
+		m.Unlock()
+		return err
+	}
+	m.vals = vals
+	m.snap = &loader.Snapshot{
+		ChangeSet: set,
+		Version:   fmt.Sprintf("%d", time.Now().Unix()),
+	}
+
+	m.Unlock()
+
+	// update watchers
+	m.update()
+
+	if len(gerr) > 0 {
+		return fmt.Errorf("source loading errors: %s", strings.Join(gerr, "\n"))
+	}
+
+	return nil
+}
+
+func (m *memory) Close() error {
+	select {
+	case <-m.exit:
+		return nil
+	default:
+		close(m.exit)
+	}
+	return nil
+}
+
+func (m *memory) Get(path ...string) (reader.Value, error) {
+	if !m.loaded() {
+		if err := m.Sync(); err != nil {
+			return nil, err
+		}
+	}
+
+	m.Lock()
+	defer m.Unlock()
+
+	// did sync actually work?
+	if m.vals != nil {
+		return m.vals.Get(path...), nil
+	}
+
+	// assuming vals is nil
+	// create new vals
+
+	ch := m.snap.ChangeSet
+
+	// we are truly screwed, trying to load in a hacked way
+	v, err := m.opts.Reader.Values(ch)
+	if err != nil {
+		return nil, err
+	}
+
+	// lets set it just because
+	m.vals = v
+
+	if m.vals != nil {
+		return m.vals.Get(path...), nil
+	}
+
+	// ok we're going hardcore now
+	return nil, errors.New("no values")
+}
+
+func (m *memory) Load(sources ...source.Source) error {
+	var gerrors []string
+
+	for _, source := range sources {
+		set, err := source.Read()
+		if err != nil {
+			gerrors = append(gerrors,
+				fmt.Sprintf("error loading source %s: %v",
+					source,
+					err))
+			// continue processing
+			continue
+		}
+		m.Lock()
+		m.sources = append(m.sources, source)
+		m.sets = append(m.sets, set)
+		idx := len(m.sets) - 1
+		m.Unlock()
+		go m.watch(idx, source)
+	}
+
+	if err := m.reload(); err != nil {
+		gerrors = append(gerrors, err.Error())
+	}
+
+	// Return errors
+	if len(gerrors) != 0 {
+		return errors.New(strings.Join(gerrors, "\n"))
+	}
+	return nil
+}
+
+func (m *memory) Watch(path ...string) (loader.Watcher, error) {
+	value, err := m.Get(path...)
+	if err != nil {
+		return nil, err
+	}
+
+	m.Lock()
+
+	w := &watcher{
+		exit:    make(chan bool),
+		path:    path,
+		value:   value,
+		reader:  m.opts.Reader,
+		updates: make(chan reader.Value, 1),
+	}
+
+	id := m.idx
+	m.watchers[id] = w
+	m.idx++
+
+	m.Unlock()
+
+	go func() {
+		<-w.exit
+		m.Lock()
+		delete(m.watchers, id)
+		m.Unlock()
+	}()
+
+	return w, nil
+}
+
+func (m *memory) String() string {
+	return "memory"
+}
+
+func (w *watcher) Next() (*loader.Snapshot, error) {
+	for {
+		select {
+		case <-w.exit:
+			return nil, errors.New("watcher stopped")
+		case v := <-w.updates:
+			if bytes.Equal(w.value.Bytes(), v.Bytes()) {
+				continue
+			}
+			w.value = v
+
+			cs := &source.ChangeSet{
+				Data:      v.Bytes(),
+				Format:    w.reader.String(),
+				Source:    "memory",
+				Timestamp: time.Now(),
+			}
+			cs.Sum()
+
+			return &loader.Snapshot{
+				ChangeSet: cs,
+				Version:   fmt.Sprintf("%d", time.Now().Unix()),
+			}, nil
+		}
+	}
+}
+
+func (w *watcher) Stop() error {
+	select {
+	case <-w.exit:
+	default:
+		close(w.exit)
+	}
+	return nil
+}
+
+func NewLoader(opts ...loader.Option) loader.Loader {
+	options := loader.Options{
+		Reader: json.NewReader(),
+	}
+
+	for _, o := range opts {
+		o(&options)
+	}
+
+	m := &memory{
+		exit:     make(chan bool),
+		opts:     options,
+		watchers: make(map[int]*watcher),
+		sources:  options.Source,
+	}
+
+	for i, s := range options.Source {
+		go m.watch(i, s)
+	}
+
+	return m
+}

config/loader/memory/options.go

@@ -0,0 +1,21 @@
+package memory
+
+import (
+	"github.com/micro/go-micro/config/loader"
+	"github.com/micro/go-micro/config/reader"
+	"github.com/micro/go-micro/config/source"
+)
+
+// WithSource appends a source to list of sources
+func WithSource(s source.Source) loader.Option {
+	return func(o *loader.Options) {
+		o.Source = append(o.Source, s)
+	}
+}
+
+// WithReader sets the config reader
+func WithReader(r reader.Reader) loader.Option {
+	return func(o *loader.Options) {
+		o.Reader = r
+	}
+}

config/options.go

@@ -0,0 +1,28 @@
+package config
+
+import (
+	"github.com/micro/go-micro/config/loader"
+	"github.com/micro/go-micro/config/reader"
+	"github.com/micro/go-micro/config/source"
+)
+
+// WithLoader sets the loader for manager config
+func WithLoader(l loader.Loader) Option {
+	return func(o *Options) {
+		o.Loader = l
+	}
+}
+
+// WithSource appends a source to list of sources
+func WithSource(s source.Source) Option {
+	return func(o *Options) {
+		o.Source = append(o.Source, s)
+	}
+}
+
+// WithReader sets the config reader
+func WithReader(r reader.Reader) Option {
+	return func(o *Options) {
+		o.Reader = r
+	}
+}

config/reader/json/json.go

@@ -0,0 +1,83 @@
+package json
+
+import (
+	"errors"
+	"time"
+
+	"github.com/imdario/mergo"
+	"github.com/micro/go-micro/config/encoder"
+	"github.com/micro/go-micro/config/encoder/json"
+	"github.com/micro/go-micro/config/reader"
+	"github.com/micro/go-micro/config/source"
+)
+
+type jsonReader struct {
+	opts reader.Options
+	json encoder.Encoder
+}
+
+func (j *jsonReader) Merge(changes ...*source.ChangeSet) (*source.ChangeSet, error) {
+	var merged map[string]interface{}
+
+	for _, m := range changes {
+		if m == nil {
+			continue
+		}
+
+		if len(m.Data) == 0 {
+			continue
+		}
+
+		codec, ok := j.opts.Encoding[m.Format]
+		if !ok {
+			// fallback
+			codec = j.json
+		}
+
+		var data map[string]interface{}
+		if err := codec.Decode(m.Data, &data); err != nil {
+			return nil, err
+		}
+		if err := mergo.Map(&merged, data, mergo.WithOverride); err != nil {
+			return nil, err
+		}
+	}
+
+	b, err := j.json.Encode(merged)
+	if err != nil {
+		return nil, err
+	}
+
+	cs := &source.ChangeSet{
+		Timestamp: time.Now(),
+		Data:      b,
+		Source:    "json",
+		Format:    j.json.String(),
+	}
+	cs.Checksum = cs.Sum()
+
+	return cs, nil
+}
+
+func (j *jsonReader) Values(ch *source.ChangeSet) (reader.Values, error) {
+	if ch == nil {
+		return nil, errors.New("changeset is nil")
+	}
+	if ch.Format != "json" {
+		return nil, errors.New("unsupported format")
+	}
+	return newValues(ch)
+}
+
+func (j *jsonReader) String() string {
+	return "json"
+}
+
+// NewReader creates a json reader
+func NewReader(opts ...reader.Option) reader.Reader {
+	options := reader.NewOptions(opts...)
+	return &jsonReader{
+		json: json.NewEncoder(),
+		opts: options,
+	}
+}

config/reader/json/json_test.go

@@ -0,0 +1,43 @@
+package json
+
+import (
+	"testing"
+
+	"github.com/micro/go-micro/config/source"
+)
+
+func TestReader(t *testing.T) {
+	data := []byte(`{"foo": "bar", "baz": {"bar": "cat"}}`)
+
+	testData := []struct {
+		path  []string
+		value string
+	}{
+		{
+			[]string{"foo"},
+			"bar",
+		},
+		{
+			[]string{"baz", "bar"},
+			"cat",
+		},
+	}
+
+	r := NewReader()
+
+	c, err := r.Merge(&source.ChangeSet{Data: data}, &source.ChangeSet{})
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	values, err := r.Values(c)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	for _, test := range testData {
+		if v := values.Get(test.path...).String(""); v != test.value {
+			t.Fatalf("Expected %s got %s for path %v", test.value, v, test.path)
+		}
+	}
+}

config/reader/json/values.go

@@ -0,0 +1,208 @@
+package json
+
+import (
+	"encoding/json"
+	"fmt"
+	"strconv"
+	"strings"
+	"time"
+
+	simple "github.com/bitly/go-simplejson"
+	"github.com/micro/go-micro/config/reader"
+	"github.com/micro/go-micro/config/source"
+)
+
+type jsonValues struct {
+	ch *source.ChangeSet
+	sj *simple.Json
+}
+
+type jsonValue struct {
+	*simple.Json
+}
+
+func newValues(ch *source.ChangeSet) (reader.Values, error) {
+	sj := simple.New()
+	data, _ := reader.ReplaceEnvVars(ch.Data)
+	if err := sj.UnmarshalJSON(data); err != nil {
+		sj.SetPath(nil, string(ch.Data))
+	}
+	return &jsonValues{ch, sj}, nil
+}
+
+func newValue(s *simple.Json) reader.Value {
+	if s == nil {
+		s = simple.New()
+	}
+	return &jsonValue{s}
+}
+
+func (j *jsonValues) Get(path ...string) reader.Value {
+	return &jsonValue{j.sj.GetPath(path...)}
+}
+
+func (j *jsonValues) Del(path ...string) {
+	// delete the tree?
+	if len(path) == 0 {
+		j.sj = simple.New()
+		return
+	}
+
+	if len(path) == 1 {
+		j.sj.Del(path[0])
+		return
+	}
+
+	vals := j.sj.GetPath(path[:len(path)-1]...)
+	vals.Del(path[len(path)-1])
+	j.sj.SetPath(path[:len(path)-1], vals.Interface())
+	return
+}
+
+func (j *jsonValues) Set(val interface{}, path ...string) {
+	j.sj.SetPath(path, val)
+}
+
+func (j *jsonValues) Bytes() []byte {
+	b, _ := j.sj.MarshalJSON()
+	return b
+}
+
+func (j *jsonValues) Map() map[string]interface{} {
+	m, _ := j.sj.Map()
+	return m
+}
+
+func (j *jsonValues) Scan(v interface{}) error {
+	b, err := j.sj.MarshalJSON()
+	if err != nil {
+		return err
+	}
+	return json.Unmarshal(b, v)
+}
+
+func (j *jsonValues) String() string {
+	return "json"
+}
+
+func (j *jsonValue) Bool(def bool) bool {
+	b, err := j.Json.Bool()
+	if err == nil {
+		return b
+	}
+
+	str, ok := j.Interface().(string)
+	if !ok {
+		return def
+	}
+
+	b, err = strconv.ParseBool(str)
+	if err != nil {
+		return def
+	}
+
+	return b
+}
+
+func (j *jsonValue) Int(def int) int {
+	i, err := j.Json.Int()
+	if err == nil {
+		return i
+	}
+
+	str, ok := j.Interface().(string)
+	if !ok {
+		return def
+	}
+
+	i, err = strconv.Atoi(str)
+	if err != nil {
+		return def
+	}
+
+	return i
+}
+
+func (j *jsonValue) String(def string) string {
+	return j.Json.MustString(def)
+}
+
+func (j *jsonValue) Float64(def float64) float64 {
+	f, err := j.Json.Float64()
+	if err == nil {
+		return f
+	}
+
+	str, ok := j.Interface().(string)
+	if !ok {
+		return def
+	}
+
+	f, err = strconv.ParseFloat(str, 64)
+	if err != nil {
+		return def
+	}
+
+	return f
+}
+
+func (j *jsonValue) Duration(def time.Duration) time.Duration {
+	v, err := j.Json.String()
+	if err != nil {
+		return def
+	}
+
+	value, err := time.ParseDuration(v)
+	if err != nil {
+		return def
+	}
+
+	return value
+}
+
+func (j *jsonValue) StringSlice(def []string) []string {
+	v, err := j.Json.String()
+	if err == nil {
+		sl := strings.Split(v, ",")
+		if len(sl) > 1 {
+			return sl
+		}
+	}
+	return j.Json.MustStringArray(def)
+}
+
+func (j *jsonValue) StringMap(def map[string]string) map[string]string {
+	m, err := j.Json.Map()
+	if err != nil {
+		return def
+	}
+
+	res := map[string]string{}
+
+	for k, v := range m {
+		res[k] = fmt.Sprintf("%v", v)
+	}
+
+	return res
+}
+
+func (j *jsonValue) Scan(v interface{}) error {
+	b, err := j.Json.MarshalJSON()
+	if err != nil {
+		return err
+	}
+	return json.Unmarshal(b, v)
+}
+
+func (j *jsonValue) Bytes() []byte {
+	b, err := j.Json.Bytes()
+	if err != nil {
+		// try return marshalled
+		b, err = j.Json.MarshalJSON()
+		if err != nil {
+			return []byte{}
+		}
+		return b
+	}
+	return b
+}

config/reader/json/values_test.go

@@ -0,0 +1,39 @@
+package json
+
+import (
+	"testing"
+
+	"github.com/micro/go-micro/config/source"
+)
+
+func TestValues(t *testing.T) {
+	data := []byte(`{"foo": "bar", "baz": {"bar": "cat"}}`)
+
+	testData := []struct {
+		path  []string
+		value string
+	}{
+		{
+			[]string{"foo"},
+			"bar",
+		},
+		{
+			[]string{"baz", "bar"},
+			"cat",
+		},
+	}
+
+	values, err := newValues(&source.ChangeSet{
+		Data: data,
+	})
+
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	for _, test := range testData {
+		if v := values.Get(test.path...).String(""); v != test.value {
+			t.Fatalf("Expected %s got %s for path %v", test.value, v, test.path)
+		}
+	}
+}

config/reader/options.go

@@ -0,0 +1,42 @@
+package reader
+
+import (
+	"github.com/micro/go-micro/config/encoder"
+	"github.com/micro/go-micro/config/encoder/hcl"
+	"github.com/micro/go-micro/config/encoder/json"
+	"github.com/micro/go-micro/config/encoder/toml"
+	"github.com/micro/go-micro/config/encoder/xml"
+	"github.com/micro/go-micro/config/encoder/yaml"
+)
+
+type Options struct {
+	Encoding map[string]encoder.Encoder
+}
+
+type Option func(o *Options)
+
+func NewOptions(opts ...Option) Options {
+	options := Options{
+		Encoding: map[string]encoder.Encoder{
+			"json": json.NewEncoder(),
+			"yaml": yaml.NewEncoder(),
+			"toml": toml.NewEncoder(),
+			"xml":  xml.NewEncoder(),
+			"hcl":  hcl.NewEncoder(),
+			"yml":  yaml.NewEncoder(),
+		},
+	}
+	for _, o := range opts {
+		o(&options)
+	}
+	return options
+}
+
+func WithEncoder(e encoder.Encoder) Option {
+	return func(o *Options) {
+		if o.Encoding == nil {
+			o.Encoding = make(map[string]encoder.Encoder)
+		}
+		o.Encoding[e.String()] = e
+	}
+}

config/reader/preprocessor.go

@@ -0,0 +1,23 @@
+package reader
+
+import (
+	"os"
+	"regexp"
+)
+
+func ReplaceEnvVars(raw []byte) ([]byte, error) {
+	re := regexp.MustCompile(`\$\{([A-Za-z0-9_]+)\}`)
+	if re.Match(raw) {
+		dataS := string(raw)
+		res := re.ReplaceAllStringFunc(dataS, replaceEnvVars)
+		return []byte(res), nil
+	} else {
+		return raw, nil
+	}
+}
+
+func replaceEnvVars(element string) string {
+	v := element[2 : len(element)-1]
+	el := os.Getenv(v)
+	return el
+}

config/reader/preprocessor_test.go

@@ -0,0 +1,73 @@
+package reader
+
+import (
+	"os"
+	"strings"
+	"testing"
+)
+
+func TestReplaceEnvVars(t *testing.T) {
+	os.Setenv("myBar", "cat")
+	os.Setenv("MYBAR", "cat")
+	os.Setenv("my_Bar", "cat")
+	os.Setenv("myBar_", "cat")
+
+	testData := []struct {
+		expected string
+		data     []byte
+	}{
+		// Right use cases
+		{
+			`{"foo": "bar", "baz": {"bar": "cat"}}`,
+			[]byte(`{"foo": "bar", "baz": {"bar": "${myBar}"}}`),
+		},
+		{
+			`{"foo": "bar", "baz": {"bar": "cat"}}`,
+			[]byte(`{"foo": "bar", "baz": {"bar": "${MYBAR}"}}`),
+		},
+		{
+			`{"foo": "bar", "baz": {"bar": "cat"}}`,
+			[]byte(`{"foo": "bar", "baz": {"bar": "${my_Bar}"}}`),
+		},
+		{
+			`{"foo": "bar", "baz": {"bar": "cat"}}`,
+			[]byte(`{"foo": "bar", "baz": {"bar": "${myBar_}"}}`),
+		},
+		// Wrong use cases
+		{
+			`{"foo": "bar", "baz": {"bar": "${myBar-}"}}`,
+			[]byte(`{"foo": "bar", "baz": {"bar": "${myBar-}"}}`),
+		},
+		{
+			`{"foo": "bar", "baz": {"bar": "${}"}}`,
+			[]byte(`{"foo": "bar", "baz": {"bar": "${}"}}`),
+		},
+		{
+			`{"foo": "bar", "baz": {"bar": "$sss}"}}`,
+			[]byte(`{"foo": "bar", "baz": {"bar": "$sss}"}}`),
+		},
+		{
+			`{"foo": "bar", "baz": {"bar": "${sss"}}`,
+			[]byte(`{"foo": "bar", "baz": {"bar": "${sss"}}`),
+		},
+		{
+			`{"foo": "bar", "baz": {"bar": "{something}"}}`,
+			[]byte(`{"foo": "bar", "baz": {"bar": "{something}"}}`),
+		},
+		// Use cases without replace env vars
+		{
+			`{"foo": "bar", "baz": {"bar": "cat"}}`,
+			[]byte(`{"foo": "bar", "baz": {"bar": "cat"}}`),
+		},
+	}
+
+	for _, test := range testData {
+		res, err := ReplaceEnvVars(test.data)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if strings.Compare(test.expected, string(res)) != 0 {
+			t.Fatalf("Expected %s got %s", test.expected, res)
+		}
+	}
+}

config/reader/reader.go

@@ -0,0 +1,36 @@
+// Package reader parses change sets and provides config values
+package reader
+
+import (
+	"time"
+
+	"github.com/micro/go-micro/config/source"
+)
+
+// Reader is an interface for merging changesets
+type Reader interface {
+	Merge(...*source.ChangeSet) (*source.ChangeSet, error)
+	Values(*source.ChangeSet) (Values, error)
+	String() string
+}
+
+// Values is returned by the reader
+type Values interface {
+	Bytes() []byte
+	Get(path ...string) Value
+	Map() map[string]interface{}
+	Scan(v interface{}) error
+}
+
+// Value represents a value of any type
+type Value interface {
+	Bool(def bool) bool
+	Int(def int) int
+	String(def string) string
+	Float64(def float64) float64
+	Duration(def time.Duration) time.Duration
+	StringSlice(def []string) []string
+	StringMap(def map[string]string) map[string]string
+	Scan(val interface{}) error
+	Bytes() []byte
+}

config/source/changeset.go

@@ -0,0 +1,13 @@
+package source
+
+import (
+	"crypto/md5"
+	"fmt"
+)
+
+// Sum returns the md5 checksum of the ChangeSet data
+func (c *ChangeSet) Sum() string {
+	h := md5.New()
+	h.Write(c.Data)
+	return fmt.Sprintf("%x", h.Sum(nil))
+}

config/source/cli/README.md

@@ -0,0 +1,71 @@
+# cli Source
+
+The cli source reads config from parsed flags via a cli.Context.
+
+## Format
+
+We expect the use of the `micro/cli` package. Upper case flags will be lower cased. Dashes will be used as delimiters for nesting.
+
+### Example
+
+```go
+micro.Flags(
+    cli.StringFlag{
+        Name: "database-address",
+        Value: "127.0.0.1",
+        Usage: "the db address",
+    },
+    cli.IntFlag{
+        Name: "database-port",
+        Value: 3306,
+        Usage: "the db port",
+    },
+)
+```
+
+Becomes
+
+```json
+{
+    "database": {
+        "address": "127.0.0.1",
+        "port": 3306
+    }
+}
+```
+
+## New and Load Source
+
+Because a cli.Context is needed to retrieve the flags and their values, it is recommended to build your source from within a cli.Action.
+
+```go
+
+func main() {
+    // New Service
+    service := micro.NewService(
+        micro.Name("example"),
+        micro.Flags(
+            cli.StringFlag{
+                Name: "database-address",
+                Value: "127.0.0.1",
+                Usage: "the db address",
+            },
+        ),
+    )
+
+    var clisrc source.Source
+
+    service.Init(
+        micro.Action(func(c *cli.Context) {
+            clisrc = cli.NewSource(
+                cli.Context(c),
+	    )
+            // Alternatively, just setup your config right here
+        }),
+    )
+    
+    // ... Load and use that source ...
+    conf := config.NewConfig()
+    conf.Load(clisrc)
+}
+```

config/source/cli/cli.go

@@ -0,0 +1,146 @@
+package cli
+
+import (
+	"flag"
+	"io/ioutil"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/imdario/mergo"
+	"github.com/micro/cli"
+	"github.com/micro/go-micro/cmd"
+	"github.com/micro/go-micro/config/source"
+)
+
+type cliSource struct {
+	opts source.Options
+	ctx  *cli.Context
+}
+
+func (c *cliSource) Read() (*source.ChangeSet, error) {
+	var changes map[string]interface{}
+
+	for _, name := range c.ctx.GlobalFlagNames() {
+		tmp := toEntry(name, c.ctx.GlobalGeneric(name))
+		mergo.Map(&changes, tmp) // need to sort error handling
+	}
+
+	for _, name := range c.ctx.FlagNames() {
+		tmp := toEntry(name, c.ctx.Generic(name))
+		mergo.Map(&changes, tmp) // need to sort error handling
+	}
+
+	b, err := c.opts.Encoder.Encode(changes)
+	if err != nil {
+		return nil, err
+	}
+
+	cs := &source.ChangeSet{
+		Format:    c.opts.Encoder.String(),
+		Data:      b,
+		Timestamp: time.Now(),
+		Source:    c.String(),
+	}
+	cs.Checksum = cs.Sum()
+
+	return cs, nil
+}
+
+func toEntry(name string, v interface{}) map[string]interface{} {
+	n := strings.ToLower(name)
+	keys := strings.FieldsFunc(n, split)
+	reverse(keys)
+	tmp := make(map[string]interface{})
+	for i, k := range keys {
+		if i == 0 {
+			tmp[k] = v
+			continue
+		}
+
+		tmp = map[string]interface{}{k: tmp}
+	}
+	return tmp
+}
+
+func reverse(ss []string) {
+	for i := len(ss)/2 - 1; i >= 0; i-- {
+		opp := len(ss) - 1 - i
+		ss[i], ss[opp] = ss[opp], ss[i]
+	}
+}
+
+func split(r rune) bool {
+	return r == '-' || r == '_'
+}
+
+func (c *cliSource) Watch() (source.Watcher, error) {
+	return source.NewNoopWatcher()
+}
+
+func (c *cliSource) String() string {
+	return "cli"
+}
+
+// NewSource returns a config source for integrating parsed flags from a micro/cli.Context.
+// Hyphens are delimiters for nesting, and all keys are lowercased. The assumption is that
+// command line flags have already been parsed.
+//
+// Example:
+//      cli.StringFlag{Name: "db-host"},
+//
+//
+//      {
+//          "database": {
+//              "host": "localhost"
+//          }
+//      }
+func NewSource(opts ...source.Option) source.Source {
+	options := source.NewOptions(opts...)
+
+	var ctx *cli.Context
+
+	c, ok := options.Context.Value(contextKey{}).(*cli.Context)
+	if ok {
+		ctx = c
+	}
+
+	// no context
+	if ctx == nil {
+		// get the default app/flags
+		app := cmd.App()
+		flags := app.Flags
+
+		// create flagset
+		set := flag.NewFlagSet(app.Name, flag.ContinueOnError)
+
+		// apply flags to set
+		for _, f := range flags {
+			f.Apply(set)
+		}
+
+		// parse flags
+		set.SetOutput(ioutil.Discard)
+		set.Parse(os.Args[1:])
+
+		// normalise flags
+		normalizeFlags(app.Flags, set)
+
+		// create context
+		ctx = cli.NewContext(app, set, nil)
+	}
+
+	return &cliSource{
+		ctx:  ctx,
+		opts: options,
+	}
+}
+
+// WithContext returns a new source with the context specified.
+// The assumption is that Context is retrieved within an app.Action function.
+func WithContext(ctx *cli.Context, opts ...source.Option) source.Source {
+	return &cliSource{
+		ctx:  ctx,
+		opts: source.NewOptions(opts...),
+	}
+}

config/source/cli/cli_test.go

@@ -0,0 +1,65 @@
+package cli
+
+import (
+	"encoding/json"
+	"os"
+	"testing"
+
+	"github.com/micro/cli"
+	"github.com/micro/go-micro/cmd"
+	"github.com/micro/go-micro/config/source"
+)
+
+func test(t *testing.T, withContext bool) {
+	var src source.Source
+
+	// setup app
+	app := cmd.App()
+	app.Name = "testapp"
+	app.Flags = []cli.Flag{
+		cli.StringFlag{Name: "db-host"},
+	}
+
+	// with context
+	if withContext {
+		// set action
+		app.Action = func(c *cli.Context) {
+			src = WithContext(c)
+		}
+
+		// run app
+		app.Run([]string{"run", "-db-host", "localhost"})
+		// no context
+	} else {
+		// set args
+		os.Args = []string{"run", "-db-host", "localhost"}
+		src = NewSource()
+	}
+
+	// test config
+	c, err := src.Read()
+	if err != nil {
+		t.Error(err)
+	}
+
+	var actual map[string]interface{}
+	if err := json.Unmarshal(c.Data, &actual); err != nil {
+		t.Error(err)
+	}
+
+	actualDB := actual["db"].(map[string]interface{})
+	if actualDB["host"] != "localhost" {
+		t.Errorf("expected localhost, got %v", actualDB["name"])
+	}
+
+}
+
+func TestCliSource(t *testing.T) {
+	// without context
+	test(t, false)
+}
+
+func TestCliSourceWithContext(t *testing.T) {
+	// with context
+	test(t, true)
+}

config/source/cli/options.go

@@ -0,0 +1,20 @@
+package cli
+
+import (
+	"context"
+
+	"github.com/micro/cli"
+	"github.com/micro/go-micro/config/source"
+)
+
+type contextKey struct{}
+
+// Context sets the cli context
+func Context(c *cli.Context) source.Option {
+	return func(o *source.Options) {
+		if o.Context == nil {
+			o.Context = context.Background()
+		}
+		o.Context = context.WithValue(o.Context, contextKey{}, c)
+	}
+}

config/source/cli/util.go

@@ -0,0 +1,50 @@
+package cli
+
+import (
+	"errors"
+	"flag"
+	"strings"
+
+	"github.com/micro/cli"
+)
+
+func copyFlag(name string, ff *flag.Flag, set *flag.FlagSet) {
+	switch ff.Value.(type) {
+	case *cli.StringSlice:
+	default:
+		set.Set(name, ff.Value.String())
+	}
+}
+
+func normalizeFlags(flags []cli.Flag, set *flag.FlagSet) error {
+	visited := make(map[string]bool)
+	set.Visit(func(f *flag.Flag) {
+		visited[f.Name] = true
+	})
+	for _, f := range flags {
+		parts := strings.Split(f.GetName(), ",")
+		if len(parts) == 1 {
+			continue
+		}
+		var ff *flag.Flag
+		for _, name := range parts {
+			name = strings.Trim(name, " ")
+			if visited[name] {
+				if ff != nil {
+					return errors.New("Cannot use two forms of the same flag: " + name + " " + ff.Name)
+				}
+				ff = set.Lookup(name)
+			}
+		}
+		if ff == nil {
+			continue
+		}
+		for _, name := range parts {
+			name = strings.Trim(name, " ")
+			if !visited[name] {
+				copyFlag(name, ff, set)
+			}
+		}
+	}
+	return nil
+}

config/source/consul/README.md

@@ -0,0 +1,49 @@
+# Consul Source
+
+The consul source reads config from consul key/values
+
+## Consul Format
+
+The consul source expects keys under the default prefix `/micro/config`
+
+Values are expected to be json
+
+```
+// set database
+consul kv put micro/config/database '{"address": "10.0.0.1", "port": 3306}'
+// set cache
+consul kv put micro/config/cache '{"address": "10.0.0.2", "port": 6379}'
+```
+
+Keys are split on `/` so access becomes
+
+```
+conf.Get("micro", "config", "database")
+```
+
+## New Source
+
+Specify source with data
+
+```go
+consulSource := consul.NewSource(
+	// optionally specify consul address; default to localhost:8500
+	consul.WithAddress("10.0.0.10:8500"),
+	// optionally specify prefix; defaults to /micro/config
+	consul.WithPrefix("/my/prefix"),
+  // optionally strip the provided prefix from the keys, defaults to false
+  consul.StripPrefix(true),
+)
+```
+
+## Load Source
+
+Load the source into config
+
+```go
+// Create new config
+conf := config.NewConfig()
+
+// Load file source
+conf.Load(consulSource)
+```

config/source/consul/consul.go

@@ -0,0 +1,121 @@
+package consul
+
+import (
+	"fmt"
+	"net"
+	"time"
+
+	"github.com/hashicorp/consul/api"
+	"github.com/micro/go-micro/config/source"
+)
+
+// Currently a single consul reader
+type consul struct {
+	prefix      string
+	stripPrefix string
+	addr        string
+	opts        source.Options
+	client      *api.Client
+}
+
+var (
+	// DefaultPrefix is the prefix that consul keys will be assumed to have if you
+	// haven't specified one
+	DefaultPrefix = "/micro/config/"
+)
+
+func (c *consul) Read() (*source.ChangeSet, error) {
+	kv, _, err := c.client.KV().List(c.prefix, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	if kv == nil || len(kv) == 0 {
+		return nil, fmt.Errorf("source not found: %s", c.prefix)
+	}
+
+	data, err := makeMap(c.opts.Encoder, kv, c.stripPrefix)
+	if err != nil {
+		return nil, fmt.Errorf("error reading data: %v", err)
+	}
+
+	b, err := c.opts.Encoder.Encode(data)
+	if err != nil {
+		return nil, fmt.Errorf("error reading source: %v", err)
+	}
+
+	cs := &source.ChangeSet{
+		Timestamp: time.Now(),
+		Format:    c.opts.Encoder.String(),
+		Source:    c.String(),
+		Data:      b,
+	}
+	cs.Checksum = cs.Sum()
+
+	return cs, nil
+}
+
+func (c *consul) String() string {
+	return "consul"
+}
+
+func (c *consul) Watch() (source.Watcher, error) {
+	w, err := newWatcher(c.prefix, c.addr, c.String(), c.stripPrefix, c.opts.Encoder)
+	if err != nil {
+		return nil, err
+	}
+	return w, nil
+}
+
+// NewSource creates a new consul source
+func NewSource(opts ...source.Option) source.Source {
+	options := source.NewOptions(opts...)
+
+	// use default config
+	config := api.DefaultConfig()
+
+	// check if there are any addrs
+	a, ok := options.Context.Value(addressKey{}).(string)
+	if ok {
+		addr, port, err := net.SplitHostPort(a)
+		if ae, ok := err.(*net.AddrError); ok && ae.Err == "missing port in address" {
+			port = "8500"
+			addr = a
+			config.Address = fmt.Sprintf("%s:%s", addr, port)
+		} else if err == nil {
+			config.Address = fmt.Sprintf("%s:%s", addr, port)
+		}
+	}
+
+	dc, ok := options.Context.Value(dcKey{}).(string)
+	if ok {
+		config.Datacenter = dc
+	}
+
+	token, ok := options.Context.Value(tokenKey{}).(string)
+	if ok {
+		config.Token = token
+	}
+
+	// create the client
+	client, _ := api.NewClient(config)
+
+	prefix := DefaultPrefix
+	sp := ""
+	f, ok := options.Context.Value(prefixKey{}).(string)
+	if ok {
+		prefix = f
+	}
+
+	if b, ok := options.Context.Value(stripPrefixKey{}).(bool); ok && b {
+		sp = prefix
+	}
+
+	return &consul{
+		prefix:      prefix,
+		stripPrefix: sp,
+		addr:        config.Address,
+		opts:        options,
+		client:      client,
+	}
+}

config/source/consul/options.go

@@ -0,0 +1,63 @@
+package consul
+
+import (
+	"context"
+
+	"github.com/micro/go-micro/config/source"
+)
+
+type addressKey struct{}
+type prefixKey struct{}
+type stripPrefixKey struct{}
+type dcKey struct{}
+type tokenKey struct{}
+
+// WithAddress sets the consul address
+func WithAddress(a string) source.Option {
+	return func(o *source.Options) {
+		if o.Context == nil {
+			o.Context = context.Background()
+		}
+		o.Context = context.WithValue(o.Context, addressKey{}, a)
+	}
+}
+
+// WithPrefix sets the key prefix to use
+func WithPrefix(p string) source.Option {
+	return func(o *source.Options) {
+		if o.Context == nil {
+			o.Context = context.Background()
+		}
+		o.Context = context.WithValue(o.Context, prefixKey{}, p)
+	}
+}
+
+// StripPrefix indicates whether to remove the prefix from config entries, or leave it in place.
+func StripPrefix(strip bool) source.Option {
+	return func(o *source.Options) {
+		if o.Context == nil {
+			o.Context = context.Background()
+		}
+
+		o.Context = context.WithValue(o.Context, stripPrefixKey{}, strip)
+	}
+}
+
+func WithDatacenter(p string) source.Option {
+	return func(o *source.Options) {
+		if o.Context == nil {
+			o.Context = context.Background()
+		}
+		o.Context = context.WithValue(o.Context, dcKey{}, p)
+	}
+}
+
+// WithToken sets the key token to use
+func WithToken(p string) source.Option {
+	return func(o *source.Options) {
+		if o.Context == nil {
+			o.Context = context.Background()
+		}
+		o.Context = context.WithValue(o.Context, tokenKey{}, p)
+	}
+}

config/source/consul/util.go

@@ -0,0 +1,49 @@
+package consul
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/hashicorp/consul/api"
+	"github.com/micro/go-micro/config/encoder"
+)
+
+func makeMap(e encoder.Encoder, kv api.KVPairs, stripPrefix string) (map[string]interface{}, error) {
+	data := make(map[string]interface{})
+
+	// consul guarantees lexicographic order, so no need to sort
+	for _, v := range kv {
+		pathString := strings.TrimPrefix(strings.TrimPrefix(v.Key, stripPrefix), "/")
+		var val map[string]interface{}
+
+		// ensure a valid value is stored at this location
+		if len(v.Value) > 0 {
+			if err := e.Decode(v.Value, &val); err != nil {
+				return nil, fmt.Errorf("faild decode value. path: %s, error: %s", pathString, err)
+			}
+		}
+
+		// set target at the root
+		target := data
+
+		// then descend to the target location, creating as we go, if need be
+		if pathString != "" {
+			path := strings.Split(pathString, "/")
+			// find (or create) the location we want to put this value at
+			for _, dir := range path {
+				if _, ok := target[dir]; !ok {
+					target[dir] = make(map[string]interface{})
+				}
+				target = target[dir].(map[string]interface{})
+			}
+
+		}
+
+		// copy over the keys from the value
+		for k := range val {
+			target[k] = val[k]
+		}
+	}
+
+	return data, nil
+}

config/source/consul/watcher.go

@@ -0,0 +1,96 @@
+package consul
+
+import (
+	"errors"
+	"time"
+
+	"github.com/hashicorp/consul/api"
+	"github.com/hashicorp/consul/api/watch"
+	"github.com/micro/go-micro/config/encoder"
+	"github.com/micro/go-micro/config/source"
+)
+
+type watcher struct {
+	e           encoder.Encoder
+	name        string
+	stripPrefix string
+
+	wp   *watch.Plan
+	ch   chan *source.ChangeSet
+	exit chan bool
+}
+
+func newWatcher(key, addr, name, stripPrefix string, e encoder.Encoder) (source.Watcher, error) {
+	w := &watcher{
+		e:           e,
+		name:        name,
+		stripPrefix: stripPrefix,
+		ch:          make(chan *source.ChangeSet),
+		exit:        make(chan bool),
+	}
+
+	wp, err := watch.Parse(map[string]interface{}{"type": "keyprefix", "prefix": key})
+	if err != nil {
+		return nil, err
+	}
+
+	wp.Handler = w.handle
+
+	// wp.Run is a blocking call and will prevent newWatcher from returning
+	go wp.Run(addr)
+
+	w.wp = wp
+
+	return w, nil
+}
+
+func (w *watcher) handle(idx uint64, data interface{}) {
+	if data == nil {
+		return
+	}
+
+	kvs, ok := data.(api.KVPairs)
+	if !ok {
+		return
+	}
+
+	d, err := makeMap(w.e, kvs, w.stripPrefix)
+	if err != nil {
+		return
+	}
+
+	b, err := w.e.Encode(d)
+	if err != nil {
+		return
+	}
+
+	cs := &source.ChangeSet{
+		Timestamp: time.Now(),
+		Format:    w.e.String(),
+		Source:    w.name,
+		Data:      b,
+	}
+	cs.Checksum = cs.Sum()
+
+	w.ch <- cs
+}
+
+func (w *watcher) Next() (*source.ChangeSet, error) {
+	select {
+	case cs := <-w.ch:
+		return cs, nil
+	case <-w.exit:
+		return nil, errors.New("watcher stopped")
+	}
+}
+
+func (w *watcher) Stop() error {
+	select {
+	case <-w.exit:
+		return nil
+	default:
+		w.wp.Stop()
+		close(w.exit)
+	}
+	return nil
+}

config/source/env/README.md

@@ -0,0 +1,96 @@
+# Env Source
+
+The env source reads config from environment variables
+
+## Format
+
+We expect environment variables to be in the standard format of FOO=bar
+
+Keys are converted to lowercase and split on underscore.
+
+
+### Example
+
+```
+DATABASE_ADDRESS=127.0.0.1
+DATABASE_PORT=3306
+```
+
+Becomes
+
+```json
+{
+    "database": {
+        "address": "127.0.0.1",
+        "port": 3306
+    }
+}
+```
+
+## Prefixes
+
+Environment variables can be namespaced so we only have access to a subset. Two options are available:
+
+```
+WithPrefix(p ...string)
+WithStrippedPrefix(p ...string)
+```
+
+The former will preserve the prefix and make it a top level key in the config. The latter eliminates the prefix, reducing the nesting by one. 
+
+#### Example:
+
+Given ENVs of:
+
+```
+APP_DATABASE_ADDRESS=127.0.0.1
+APP_DATABASE_PORT=3306
+VAULT_ADDR=vault:1337
+```
+
+and a source initialized as follows:
+
+```
+src := env.NewSource(
+    env.WithPrefix("VAULT"),
+    env.WithStrippedPrefix("APP"),
+)
+```
+
+The resulting config will be:
+
+```
+{
+    "database": {
+        "address": "127.0.0.1",
+        "port": 3306
+    },
+    "vault": {
+        "addr": "vault:1337"
+    }
+}
+```
+
+
+## New Source
+
+Specify source with data
+
+```go
+src := env.NewSource(
+	// optionally specify prefix
+	env.WithPrefix("MICRO"),
+)
+```
+
+## Load Source
+
+Load the source into config
+
+```go
+// Create new config
+conf := config.NewConfig()
+
+// Load file source
+conf.Load(src)
+```

config/source/env/env.go

@@ -0,0 +1,142 @@
+package env
+
+import (
+	"os"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/imdario/mergo"
+	"github.com/micro/go-micro/config/source"
+)
+
+var (
+	DefaultPrefixes = []string{}
+)
+
+type env struct {
+	prefixes         []string
+	strippedPrefixes []string
+	opts             source.Options
+}
+
+func (e *env) Read() (*source.ChangeSet, error) {
+	var changes map[string]interface{}
+
+	for _, env := range os.Environ() {
+
+		if len(e.prefixes) > 0 || len(e.strippedPrefixes) > 0 {
+			notFound := true
+
+			if _, ok := matchPrefix(e.prefixes, env); ok {
+				notFound = false
+			}
+
+			if match, ok := matchPrefix(e.strippedPrefixes, env); ok {
+				env = strings.TrimPrefix(env, match)
+				notFound = false
+			}
+
+			if notFound {
+				continue
+			}
+		}
+
+		pair := strings.SplitN(env, "=", 2)
+		value := pair[1]
+		keys := strings.Split(strings.ToLower(pair[0]), "_")
+		reverse(keys)
+
+		tmp := make(map[string]interface{})
+		for i, k := range keys {
+			if i == 0 {
+				if intValue, err := strconv.Atoi(value); err == nil {
+					tmp[k] = intValue
+				} else if boolValue, err := strconv.ParseBool(value); err == nil {
+					tmp[k] = boolValue
+				} else {
+					tmp[k] = value
+				}
+				continue
+			}
+
+			tmp = map[string]interface{}{k: tmp}
+		}
+
+		if err := mergo.Map(&changes, tmp); err != nil {
+			return nil, err
+		}
+	}
+
+	b, err := e.opts.Encoder.Encode(changes)
+	if err != nil {
+		return nil, err
+	}
+
+	cs := &source.ChangeSet{
+		Format:    e.opts.Encoder.String(),
+		Data:      b,
+		Timestamp: time.Now(),
+		Source:    e.String(),
+	}
+	cs.Checksum = cs.Sum()
+
+	return cs, nil
+}
+
+func matchPrefix(pre []string, s string) (string, bool) {
+	for _, p := range pre {
+		if strings.HasPrefix(s, p) {
+			return p, true
+		}
+	}
+
+	return "", false
+}
+
+func reverse(ss []string) {
+	for i := len(ss)/2 - 1; i >= 0; i-- {
+		opp := len(ss) - 1 - i
+		ss[i], ss[opp] = ss[opp], ss[i]
+	}
+}
+
+func (e *env) Watch() (source.Watcher, error) {
+	return newWatcher()
+}
+
+func (e *env) String() string {
+	return "env"
+}
+
+// NewSource returns a config source for parsing ENV variables.
+// Underscores are delimiters for nesting, and all keys are lowercased.
+//
+// Example:
+//      "DATABASE_SERVER_HOST=localhost" will convert to
+//
+//      {
+//          "database": {
+//              "server": {
+//                  "host": "localhost"
+//              }
+//          }
+//      }
+func NewSource(opts ...source.Option) source.Source {
+	options := source.NewOptions(opts...)
+
+	var sp []string
+	var pre []string
+	if p, ok := options.Context.Value(strippedPrefixKey{}).([]string); ok {
+		sp = p
+	}
+
+	if p, ok := options.Context.Value(prefixKey{}).([]string); ok {
+		pre = p
+	}
+
+	if len(sp) > 0 || len(pre) > 0 {
+		pre = append(pre, DefaultPrefixes...)
+	}
+	return &env{prefixes: pre, strippedPrefixes: sp, opts: options}
+}

config/source/env/env_test.go

@@ -0,0 +1,112 @@
+package env
+
+import (
+	"encoding/json"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/micro/go-micro/config/source"
+)
+
+func TestEnv_Read(t *testing.T) {
+	expected := map[string]map[string]string{
+		"database": {
+			"host":       "localhost",
+			"password":   "password",
+			"datasource": "user:password@tcp(localhost:port)/db?charset=utf8mb4&parseTime=True&loc=Local",
+		},
+	}
+
+	os.Setenv("DATABASE_HOST", "localhost")
+	os.Setenv("DATABASE_PASSWORD", "password")
+	os.Setenv("DATABASE_DATASOURCE", "user:password@tcp(localhost:port)/db?charset=utf8mb4&parseTime=True&loc=Local")
+
+	source := NewSource()
+	c, err := source.Read()
+	if err != nil {
+		t.Error(err)
+	}
+
+	var actual map[string]interface{}
+	if err := json.Unmarshal(c.Data, &actual); err != nil {
+		t.Error(err)
+	}
+
+	actualDB := actual["database"].(map[string]interface{})
+
+	for k, v := range expected["database"] {
+		a := actualDB[k]
+
+		if a != v {
+			t.Errorf("expected %v got %v", v, a)
+		}
+	}
+}
+
+func TestEnvvar_Prefixes(t *testing.T) {
+	os.Setenv("APP_DATABASE_HOST", "localhost")
+	os.Setenv("APP_DATABASE_PASSWORD", "password")
+	os.Setenv("VAULT_ADDR", "vault:1337")
+	os.Setenv("MICRO_REGISTRY", "mdns")
+
+	var prefixtests = []struct {
+		prefixOpts   []source.Option
+		expectedKeys []string
+	}{
+		{[]source.Option{WithPrefix("APP", "MICRO")}, []string{"app", "micro"}},
+		{[]source.Option{WithPrefix("MICRO"), WithStrippedPrefix("APP")}, []string{"database", "micro"}},
+		{[]source.Option{WithPrefix("MICRO"), WithStrippedPrefix("APP")}, []string{"database", "micro"}},
+	}
+
+	for _, pt := range prefixtests {
+		source := NewSource(pt.prefixOpts...)
+
+		c, err := source.Read()
+		if err != nil {
+			t.Error(err)
+		}
+
+		var actual map[string]interface{}
+		if err := json.Unmarshal(c.Data, &actual); err != nil {
+			t.Error(err)
+		}
+
+		// assert other prefixes ignored
+		if l := len(actual); l != len(pt.expectedKeys) {
+			t.Errorf("expected %v top keys, got %v", len(pt.expectedKeys), l)
+		}
+
+		for _, k := range pt.expectedKeys {
+			if !containsKey(actual, k) {
+				t.Errorf("expected key %v, not found", k)
+			}
+		}
+	}
+}
+
+func TestEnvvar_WatchNextNoOpsUntilStop(t *testing.T) {
+	source := NewSource(WithStrippedPrefix("GOMICRO_"))
+	w, err := source.Watch()
+	if err != nil {
+		t.Error(err)
+	}
+
+	go func() {
+		time.Sleep(50 * time.Millisecond)
+		w.Stop()
+	}()
+
+	if _, err := w.Next(); err.Error() != "watcher stopped" {
+		t.Errorf("expected watcher stopped error, got %v", err)
+	}
+}
+
+func containsKey(m map[string]interface{}, s string) bool {
+	for k := range m {
+		if k == s {
+			return true
+		}
+	}
+	return false
+}

config/source/env/options.go

@@ -0,0 +1,49 @@
+package env
+
+import (
+	"context"
+
+	"strings"
+
+	"github.com/micro/go-micro/config/source"
+)
+
+type strippedPrefixKey struct{}
+type prefixKey struct{}
+
+// WithStrippedPrefix sets the environment variable prefixes to scope to.
+// These prefixes will be removed from the actual config entries.
+func WithStrippedPrefix(p ...string) source.Option {
+	return func(o *source.Options) {
+		if o.Context == nil {
+			o.Context = context.Background()
+		}
+
+		o.Context = context.WithValue(o.Context, strippedPrefixKey{}, appendUnderscore(p))
+	}
+}
+
+// WithPrefix sets the environment variable prefixes to scope to.
+// These prefixes will not be removed. Each prefix will be considered a top level config entry.
+func WithPrefix(p ...string) source.Option {
+	return func(o *source.Options) {
+		if o.Context == nil {
+			o.Context = context.Background()
+		}
+		o.Context = context.WithValue(o.Context, prefixKey{}, appendUnderscore(p))
+	}
+}
+
+func appendUnderscore(prefixes []string) []string {
+	var result []string
+	for _, p := range prefixes {
+		if !strings.HasSuffix(p, "_") {
+			result = append(result, p+"_")
+			continue
+		}
+
+		result = append(result, p)
+	}
+
+	return result
+}

config/source/env/watcher.go

@@ -0,0 +1,26 @@
+package env
+
+import (
+	"errors"
+
+	"github.com/micro/go-micro/config/source"
+)
+
+type watcher struct {
+	exit chan struct{}
+}
+
+func (w *watcher) Next() (*source.ChangeSet, error) {
+	<-w.exit
+
+	return nil, errors.New("watcher stopped")
+}
+
+func (w *watcher) Stop() error {
+	close(w.exit)
+	return nil
+}
+
+func newWatcher() (source.Watcher, error) {
+	return &watcher{exit: make(chan struct{})}, nil
+}

config/source/etcd/README.md

@@ -0,0 +1,51 @@
+# Etcd Source
+
+The etcd source reads config from etcd key/values
+
+This source supports etcd version 3 and beyond.
+
+## Etcd Format
+
+The etcd source expects keys under the default prefix `/micro/config` (prefix can be changed)
+
+Values are expected to be JSON
+
+```
+// set database
+etcdctl put /micro/config/database '{"address": "10.0.0.1", "port": 3306}'
+// set cache
+etcdctl put /micro/config/cache '{"address": "10.0.0.2", "port": 6379}'
+```
+
+Keys are split on `/` so access becomes
+
+```
+conf.Get("micro", "config", "database")
+```
+
+## New Source
+
+Specify source with data
+
+```go
+etcdSource := etcd.NewSource(
+	// optionally specify etcd address; default to localhost:8500
+	etcd.WithAddress("10.0.0.10:8500"),
+	// optionally specify prefix; defaults to /micro/config
+	etcd.WithPrefix("/my/prefix"),
+	// optionally strip the provided prefix from the keys, defaults to false
+	etcd.StripPrefix(true),
+)
+```
+
+## Load Source
+
+Load the source into config
+
+```go
+// Create new config
+conf := config.NewConfig()
+
+// Load file source
+conf.Load(etcdSource)
+```

config/source/etcd/etcd.go

@@ -0,0 +1,134 @@
+package etcd
+
+import (
+	"context"
+	"fmt"
+	"net"
+	"time"
+
+	"github.com/micro/go-micro/config/source"
+	cetcd "go.etcd.io/etcd/clientv3"
+	"go.etcd.io/etcd/mvcc/mvccpb"
+)
+
+// Currently a single etcd reader
+type etcd struct {
+	prefix      string
+	stripPrefix string
+	opts        source.Options
+	client      *cetcd.Client
+	cerr        error
+}
+
+var (
+	DefaultPrefix = "/micro/config/"
+)
+
+func (c *etcd) Read() (*source.ChangeSet, error) {
+	if c.cerr != nil {
+		return nil, c.cerr
+	}
+
+	rsp, err := c.client.Get(context.Background(), c.prefix, cetcd.WithPrefix())
+	if err != nil {
+		return nil, err
+	}
+
+	if rsp == nil || len(rsp.Kvs) == 0 {
+		return nil, fmt.Errorf("source not found: %s", c.prefix)
+	}
+
+	var kvs []*mvccpb.KeyValue
+	for _, v := range rsp.Kvs {
+		kvs = append(kvs, (*mvccpb.KeyValue)(v))
+	}
+
+	data := makeMap(c.opts.Encoder, kvs, c.stripPrefix)
+
+	b, err := c.opts.Encoder.Encode(data)
+	if err != nil {
+		return nil, fmt.Errorf("error reading source: %v", err)
+	}
+
+	cs := &source.ChangeSet{
+		Timestamp: time.Now(),
+		Source:    c.String(),
+		Data:      b,
+		Format:    c.opts.Encoder.String(),
+	}
+	cs.Checksum = cs.Sum()
+
+	return cs, nil
+}
+
+func (c *etcd) String() string {
+	return "etcd"
+}
+
+func (c *etcd) Watch() (source.Watcher, error) {
+	if c.cerr != nil {
+		return nil, c.cerr
+	}
+	cs, err := c.Read()
+	if err != nil {
+		return nil, err
+	}
+	return newWatcher(c.prefix, c.stripPrefix, c.client.Watcher, cs, c.opts)
+}
+
+func NewSource(opts ...source.Option) source.Source {
+	options := source.NewOptions(opts...)
+
+	var endpoints []string
+
+	// check if there are any addrs
+	addrs, ok := options.Context.Value(addressKey{}).([]string)
+	if ok {
+		for _, a := range addrs {
+			addr, port, err := net.SplitHostPort(a)
+			if ae, ok := err.(*net.AddrError); ok && ae.Err == "missing port in address" {
+				port = "2379"
+				addr = a
+				endpoints = append(endpoints, fmt.Sprintf("%s:%s", addr, port))
+			} else if err == nil {
+				endpoints = append(endpoints, fmt.Sprintf("%s:%s", addr, port))
+			}
+		}
+	}
+
+	if len(endpoints) == 0 {
+		endpoints = []string{"localhost:2379"}
+	}
+
+	config := cetcd.Config{
+		Endpoints: endpoints,
+	}
+
+	u, ok := options.Context.Value(authKey{}).(*authCreds)
+	if ok {
+		config.Username = u.Username
+		config.Password = u.Password
+	}
+
+	// use default config
+	client, err := cetcd.New(config)
+
+	prefix := DefaultPrefix
+	sp := ""
+	f, ok := options.Context.Value(prefixKey{}).(string)
+	if ok {
+		prefix = f
+	}
+
+	if b, ok := options.Context.Value(stripPrefixKey{}).(bool); ok && b {
+		sp = prefix
+	}
+
+	return &etcd{
+		prefix:      prefix,
+		stripPrefix: sp,
+		opts:        options,
+		client:      client,
+		cerr:        err,
+	}
+}

config/source/etcd/options.go

@@ -0,0 +1,58 @@
+package etcd
+
+import (
+	"context"
+
+	"github.com/micro/go-micro/config/source"
+)
+
+type addressKey struct{}
+type prefixKey struct{}
+type stripPrefixKey struct{}
+type authKey struct{}
+
+type authCreds struct {
+	Username string
+	Password string
+}
+
+// WithAddress sets the consul address
+func WithAddress(a ...string) source.Option {
+	return func(o *source.Options) {
+		if o.Context == nil {
+			o.Context = context.Background()
+		}
+		o.Context = context.WithValue(o.Context, addressKey{}, a)
+	}
+}
+
+// WithPrefix sets the key prefix to use
+func WithPrefix(p string) source.Option {
+	return func(o *source.Options) {
+		if o.Context == nil {
+			o.Context = context.Background()
+		}
+		o.Context = context.WithValue(o.Context, prefixKey{}, p)
+	}
+}
+
+// StripPrefix indicates whether to remove the prefix from config entries, or leave it in place.
+func StripPrefix(strip bool) source.Option {
+	return func(o *source.Options) {
+		if o.Context == nil {
+			o.Context = context.Background()
+		}
+
+		o.Context = context.WithValue(o.Context, stripPrefixKey{}, strip)
+	}
+}
+
+// Auth allows you to specify username/password
+func Auth(username, password string) source.Option {
+	return func(o *source.Options) {
+		if o.Context == nil {
+			o.Context = context.Background()
+		}
+		o.Context = context.WithValue(o.Context, authKey{}, &authCreds{Username: username, Password: password})
+	}
+}

config/source/etcd/util.go

@@ -0,0 +1,89 @@
+package etcd
+
+import (
+	"strings"
+
+	"github.com/micro/go-micro/config/encoder"
+	"go.etcd.io/etcd/clientv3"
+	"go.etcd.io/etcd/mvcc/mvccpb"
+)
+
+func makeEvMap(e encoder.Encoder, data map[string]interface{}, kv []*clientv3.Event, stripPrefix string) map[string]interface{} {
+	if data == nil {
+		data = make(map[string]interface{})
+	}
+
+	for _, v := range kv {
+		switch mvccpb.Event_EventType(v.Type) {
+		case mvccpb.DELETE:
+			data = update(e, data, (*mvccpb.KeyValue)(v.Kv), "delete", stripPrefix)
+		default:
+			data = update(e, data, (*mvccpb.KeyValue)(v.Kv), "insert", stripPrefix)
+		}
+	}
+
+	return data
+}
+
+func makeMap(e encoder.Encoder, kv []*mvccpb.KeyValue, stripPrefix string) map[string]interface{} {
+	data := make(map[string]interface{})
+
+	for _, v := range kv {
+		data = update(e, data, v, "put", stripPrefix)
+	}
+
+	return data
+}
+
+func update(e encoder.Encoder, data map[string]interface{}, v *mvccpb.KeyValue, action, stripPrefix string) map[string]interface{} {
+	// remove prefix if non empty, and ensure leading / is removed as well
+	vkey := strings.TrimPrefix(strings.TrimPrefix(string(v.Key), stripPrefix), "/")
+	// split on prefix
+	haveSplit := strings.Contains(vkey, "/")
+	keys := strings.Split(vkey, "/")
+
+	var vals interface{}
+	e.Decode(v.Value, &vals)
+
+	if !haveSplit && len(keys) == 1 {
+		switch action {
+		case "delete":
+			data = make(map[string]interface{})
+		default:
+			v, ok := vals.(map[string]interface{})
+			if ok {
+				data = v
+			}
+		}
+		return data
+	}
+
+	// set data for first iteration
+	kvals := data
+	// iterate the keys and make maps
+	for i, k := range keys {
+		kval, ok := kvals[k].(map[string]interface{})
+		if !ok {
+			// create next map
+			kval = make(map[string]interface{})
+			// set it
+			kvals[k] = kval
+		}
+
+		// last key: write vals
+		if l := len(keys) - 1; i == l {
+			switch action {
+			case "delete":
+				delete(kvals, k)
+			default:
+				kvals[k] = vals
+			}
+			break
+		}
+
+		// set kvals for next iterator
+		kvals = kval
+	}
+
+	return data
+}

config/source/etcd/watcher.go

@@ -0,0 +1,113 @@
+package etcd
+
+import (
+	"context"
+	"errors"
+	"sync"
+	"time"
+
+	"github.com/micro/go-micro/config/source"
+	cetcd "go.etcd.io/etcd/clientv3"
+)
+
+type watcher struct {
+	opts        source.Options
+	name        string
+	stripPrefix string
+
+	sync.RWMutex
+	cs *source.ChangeSet
+
+	ch   chan *source.ChangeSet
+	exit chan bool
+}
+
+func newWatcher(key, strip string, wc cetcd.Watcher, cs *source.ChangeSet, opts source.Options) (source.Watcher, error) {
+	w := &watcher{
+		opts:        opts,
+		name:        "etcd",
+		stripPrefix: strip,
+		cs:          cs,
+		ch:          make(chan *source.ChangeSet),
+		exit:        make(chan bool),
+	}
+
+	ch := wc.Watch(context.Background(), key, cetcd.WithPrefix())
+
+	go w.run(wc, ch)
+
+	return w, nil
+}
+
+func (w *watcher) handle(evs []*cetcd.Event) {
+	w.RLock()
+	data := w.cs.Data
+	w.RUnlock()
+
+	var vals map[string]interface{}
+
+	// unpackage existing changeset
+	if err := w.opts.Encoder.Decode(data, &vals); err != nil {
+		return
+	}
+
+	// update base changeset
+	d := makeEvMap(w.opts.Encoder, vals, evs, w.stripPrefix)
+
+	// pack the changeset
+	b, err := w.opts.Encoder.Encode(d)
+	if err != nil {
+		return
+	}
+
+	// create new changeset
+	cs := &source.ChangeSet{
+		Timestamp: time.Now(),
+		Source:    w.name,
+		Data:      b,
+		Format:    w.opts.Encoder.String(),
+	}
+	cs.Checksum = cs.Sum()
+
+	// set base change set
+	w.Lock()
+	w.cs = cs
+	w.Unlock()
+
+	// send update
+	w.ch <- cs
+}
+
+func (w *watcher) run(wc cetcd.Watcher, ch cetcd.WatchChan) {
+	for {
+		select {
+		case rsp, ok := <-ch:
+			if !ok {
+				return
+			}
+			w.handle(rsp.Events)
+		case <-w.exit:
+			wc.Close()
+			return
+		}
+	}
+}
+
+func (w *watcher) Next() (*source.ChangeSet, error) {
+	select {
+	case cs := <-w.ch:
+		return cs, nil
+	case <-w.exit:
+		return nil, errors.New("watcher stopped")
+	}
+}
+
+func (w *watcher) Stop() error {
+	select {
+	case <-w.exit:
+		return nil
+	default:
+		close(w.exit)
+	}
+	return nil
+}

config/source/file/README.md

@@ -0,0 +1,70 @@
+# File Source
+
+The file source reads config from a file. 
+
+It uses the File extension to determine the Format e.g `config.yaml` has the yaml format. 
+It does not make use of encoders or interpet the file data. If a file extension is not present 
+the source Format will default to the Encoder in options.
+
+## Example
+
+A config file format in json
+
+```json
+{
+    "hosts": {
+        "database": {
+            "address": "10.0.0.1",
+            "port": 3306
+        },
+        "cache": {
+            "address": "10.0.0.2",
+            "port": 6379
+        }
+    }
+}
+```
+
+## New Source
+
+Specify file source with path to file. Path is optional and will default to `config.json`
+
+```go
+fileSource := file.NewSource(
+	file.WithPath("/tmp/config.json"),
+)
+```
+
+## File Format
+
+To load different file formats e.g yaml, toml, xml simply specify them with their extension
+
+```
+fileSource := file.NewSource(
+        file.WithPath("/tmp/config.yaml"),
+)
+```
+
+If you want to specify a file without extension, ensure you set the encoder to the same format
+
+```
+e := toml.NewEncoder()
+
+fileSource := file.NewSource(
+        file.WithPath("/tmp/config"),
+	source.WithEncoder(e),
+)
+```
+
+## Load Source
+
+Load the source into config
+
+```go
+// Create new config
+conf := config.NewConfig()
+
+// Load file source
+conf.Load(fileSource)
+```
+

config/source/file/file.go

@@ -0,0 +1,65 @@
+// Package file is a file source. Expected format is json
+package file
+
+import (
+	"io/ioutil"
+	"os"
+
+	"github.com/micro/go-micro/config/source"
+)
+
+type file struct {
+	path string
+	opts source.Options
+}
+
+var (
+	DefaultPath = "config.json"
+)
+
+func (f *file) Read() (*source.ChangeSet, error) {
+	fh, err := os.Open(f.path)
+	if err != nil {
+		return nil, err
+	}
+	defer fh.Close()
+	b, err := ioutil.ReadAll(fh)
+	if err != nil {
+		return nil, err
+	}
+	info, err := fh.Stat()
+	if err != nil {
+		return nil, err
+	}
+
+	cs := &source.ChangeSet{
+		Format:    format(f.path, f.opts.Encoder),
+		Source:    f.String(),
+		Timestamp: info.ModTime(),
+		Data:      b,
+	}
+	cs.Checksum = cs.Sum()
+
+	return cs, nil
+}
+
+func (f *file) String() string {
+	return "file"
+}
+
+func (f *file) Watch() (source.Watcher, error) {
+	if _, err := os.Stat(f.path); err != nil {
+		return nil, err
+	}
+	return newWatcher(f)
+}
+
+func NewSource(opts ...source.Option) source.Source {
+	options := source.NewOptions(opts...)
+	path := DefaultPath
+	f, ok := options.Context.Value(filePathKey{}).(string)
+	if ok {
+		path = f
+	}
+	return &file{opts: options, path: path}
+}

config/source/file/file_test.go

@@ -0,0 +1,37 @@
+package file
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+)
+
+func TestFile(t *testing.T) {
+	data := []byte(`{"foo": "bar"}`)
+	path := filepath.Join(os.TempDir(), fmt.Sprintf("file.%d", time.Now().UnixNano()))
+	fh, err := os.Create(path)
+	if err != nil {
+		t.Error(err)
+	}
+	defer func() {
+		fh.Close()
+		os.Remove(path)
+	}()
+
+	_, err = fh.Write(data)
+	if err != nil {
+		t.Error(err)
+	}
+
+	f := NewSource(WithPath(path))
+	c, err := f.Read()
+	if err != nil {
+		t.Error(err)
+	}
+	t.Logf("%+v", c)
+	if string(c.Data) != string(data) {
+		t.Error("data from file does not match")
+	}
+}

config/source/file/format.go

@@ -0,0 +1,15 @@
+package file
+
+import (
+	"strings"
+
+	"github.com/micro/go-micro/config/encoder"
+)
+
+func format(p string, e encoder.Encoder) string {
+	parts := strings.Split(p, ".")
+	if len(parts) > 1 {
+		return parts[len(parts)-1]
+	}
+	return e.String()
+}

config/source/file/format_test.go

@@ -0,0 +1,31 @@
+package file
+
+import (
+	"testing"
+
+	"github.com/micro/go-micro/config/source"
+)
+
+func TestFormat(t *testing.T) {
+	opts := source.NewOptions()
+	e := opts.Encoder
+
+	testCases := []struct {
+		p string
+		f string
+	}{
+		{"/foo/bar.json", "json"},
+		{"/foo/bar.yaml", "yaml"},
+		{"/foo/bar.xml", "xml"},
+		{"/foo/bar.conf.ini", "ini"},
+		{"conf", e.String()},
+	}
+
+	for _, d := range testCases {
+		f := format(d.p, e)
+		if f != d.f {
+			t.Fatalf("%s: expected %s got %s", d.p, d.f, f)
+		}
+	}
+
+}

config/source/file/options.go

@@ -0,0 +1,19 @@
+package file
+
+import (
+	"context"
+
+	"github.com/micro/go-micro/config/source"
+)
+
+type filePathKey struct{}
+
+// WithPath sets the path to file
+func WithPath(p string) source.Option {
+	return func(o *source.Options) {
+		if o.Context == nil {
+			o.Context = context.Background()
+		}
+		o.Context = context.WithValue(o.Context, filePathKey{}, p)
+	}
+}

config/source/file/watcher.go

@@ -0,0 +1,66 @@
+package file
+
+import (
+	"errors"
+	"os"
+
+	"github.com/fsnotify/fsnotify"
+	"github.com/micro/go-micro/config/source"
+)
+
+type watcher struct {
+	f *file
+
+	fw   *fsnotify.Watcher
+	exit chan bool
+}
+
+func newWatcher(f *file) (source.Watcher, error) {
+	fw, err := fsnotify.NewWatcher()
+	if err != nil {
+		return nil, err
+	}
+
+	fw.Add(f.path)
+
+	return &watcher{
+		f:    f,
+		fw:   fw,
+		exit: make(chan bool),
+	}, nil
+}
+
+func (w *watcher) Next() (*source.ChangeSet, error) {
+	// is it closed?
+	select {
+	case <-w.exit:
+		return nil, errors.New("watcher stopped")
+	default:
+	}
+
+	// try get the event
+	select {
+	case event, _ := <-w.fw.Events:
+		if event.Op == fsnotify.Rename {
+			// check existence of file, and add watch again
+			_, err := os.Stat(event.Name)
+			if err == nil || os.IsExist(err) {
+				w.fw.Add(event.Name)
+			}
+		}
+
+		c, err := w.f.Read()
+		if err != nil {
+			return nil, err
+		}
+		return c, nil
+	case err := <-w.fw.Errors:
+		return nil, err
+	case <-w.exit:
+		return nil, errors.New("watcher stopped")
+	}
+}
+
+func (w *watcher) Stop() error {
+	return w.fw.Close()
+}

config/source/flag/README.md

@@ -0,0 +1,47 @@
+# Flag Source
+
+The flag source reads config from flags
+
+## Format
+
+We expect the use of the `flag` package. Upper case flags will be lower cased. Dashes will be used as delimiters.
+
+### Example
+
+```
+dbAddress := flag.String("database_address", "127.0.0.1", "the db address")
+dbPort := flag.Int("database_port", 3306, "the db port)
+```
+
+Becomes
+
+```json
+{
+    "database": {
+        "address": "127.0.0.1",
+        "port": 3306
+    }
+}
+```
+
+## New Source
+
+```go
+flagSource := flag.NewSource(
+	// optionally enable reading of unset flags and their default
+	// values into config, defaults to false
+	IncludeUnset(true)
+)
+```
+
+## Load Source
+
+Load the source into config
+
+```go
+// Create new config
+conf := config.NewConfig()
+
+// Load file source
+conf.Load(flagSource)
+```

config/source/flag/flag.go

@@ -0,0 +1,97 @@
+package flag
+
+import (
+	"errors"
+	"flag"
+	"github.com/imdario/mergo"
+	"github.com/micro/go-micro/config/source"
+	"strings"
+	"time"
+)
+
+type flagsrc struct {
+	opts source.Options
+}
+
+func (fs *flagsrc) Read() (*source.ChangeSet, error) {
+	if !flag.Parsed() {
+		return nil, errors.New("flags not parsed")
+	}
+
+	var changes map[string]interface{}
+
+	visitFn := func(f *flag.Flag) {
+		n := strings.ToLower(f.Name)
+		keys := strings.FieldsFunc(n, split)
+		reverse(keys)
+
+		tmp := make(map[string]interface{})
+		for i, k := range keys {
+			if i == 0 {
+				tmp[k] = f.Value
+				continue
+			}
+
+			tmp = map[string]interface{}{k: tmp}
+		}
+
+		mergo.Map(&changes, tmp) // need to sort error handling
+		return
+	}
+
+	unset, ok := fs.opts.Context.Value(includeUnsetKey{}).(bool)
+	if ok && unset {
+		flag.VisitAll(visitFn)
+	} else {
+		flag.Visit(visitFn)
+	}
+
+	b, err := fs.opts.Encoder.Encode(changes)
+	if err != nil {
+		return nil, err
+	}
+
+	cs := &source.ChangeSet{
+		Format:    fs.opts.Encoder.String(),
+		Data:      b,
+		Timestamp: time.Now(),
+		Source:    fs.String(),
+	}
+	cs.Checksum = cs.Sum()
+
+	return cs, nil
+}
+
+func split(r rune) bool {
+	return r == '-' || r == '_'
+}
+
+func reverse(ss []string) {
+	for i := len(ss)/2 - 1; i >= 0; i-- {
+		opp := len(ss) - 1 - i
+		ss[i], ss[opp] = ss[opp], ss[i]
+	}
+}
+
+func (fs *flagsrc) Watch() (source.Watcher, error) {
+	return source.NewNoopWatcher()
+}
+
+func (fs *flagsrc) String() string {
+	return "flag"
+}
+
+// NewSource returns a config source for integrating parsed flags.
+// Hyphens are delimiters for nesting, and all keys are lowercased.
+//
+// Example:
+//      dbhost := flag.String("database-host", "localhost", "the db host name")
+//
+//      {
+//          "database": {
+//              "host": "localhost"
+//          }
+//      }
+func NewSource(opts ...source.Option) source.Source {
+	return &flagsrc{opts: source.NewOptions(opts...)}
+}

config/source/flag/flag_test.go

@@ -0,0 +1,66 @@
+package flag
+
+import (
+	"encoding/json"
+	"flag"
+	"testing"
+)
+
+var (
+	dbuser = flag.String("database-user", "default", "db user")
+	dbhost = flag.String("database-host", "", "db host")
+	dbpw   = flag.String("database-password", "", "db pw")
+)
+
+func init() {
+	flag.Set("database-host", "localhost")
+	flag.Set("database-password", "some-password")
+	flag.Parse()
+}
+
+func TestFlagsrc_Read(t *testing.T) {
+	source := NewSource()
+	c, err := source.Read()
+	if err != nil {
+		t.Error(err)
+	}
+
+	var actual map[string]interface{}
+	if err := json.Unmarshal(c.Data, &actual); err != nil {
+		t.Error(err)
+	}
+
+	actualDB := actual["database"].(map[string]interface{})
+	if actualDB["host"] != *dbhost {
+		t.Errorf("expected %v got %v", *dbhost, actualDB["host"])
+	}
+
+	if actualDB["password"] != *dbpw {
+		t.Errorf("expected %v got %v", *dbpw, actualDB["password"])
+	}
+
+	// unset flags should not be loaded
+	if actualDB["user"] != nil {
+		t.Errorf("expected %v got %v", nil, actualDB["user"])
+	}
+}
+
+func TestFlagsrc_ReadAll(t *testing.T) {
+	source := NewSource(IncludeUnset(true))
+	c, err := source.Read()
+	if err != nil {
+		t.Error(err)
+	}
+
+	var actual map[string]interface{}
+	if err := json.Unmarshal(c.Data, &actual); err != nil {
+		t.Error(err)
+	}
+
+	actualDB := actual["database"].(map[string]interface{})
+
+	// unset flag defaults should be loaded
+	if actualDB["user"] != *dbuser {
+		t.Errorf("expected %v got %v", *dbuser, actualDB["user"])
+	}
+}

config/source/flag/options.go

@@ -0,0 +1,20 @@
+package flag
+
+import (
+	"context"
+
+	"github.com/micro/go-micro/config/source"
+)
+
+type includeUnsetKey struct{}
+
+// IncludeUnset toggles the loading of unset flags and their respective default values.
+// Default behavior is to ignore any unset flags.
+func IncludeUnset(b bool) source.Option {
+	return func(o *source.Options) {
+		if o.Context == nil {
+			o.Context = context.Background()
+		}
+		o.Context = context.WithValue(o.Context, includeUnsetKey{}, true)
+	}
+}

config/source/memory/README.md

@@ -0,0 +1,44 @@
+# Memory Source
+
+The memory source provides in-memory data as a source
+
+## Memory Format
+
+The expected data format is json
+
+```json
+data := []byte(`{
+    "hosts": {
+        "database": {
+            "address": "10.0.0.1",
+            "port": 3306
+        },
+        "cache": {
+            "address": "10.0.0.2",
+            "port": 6379
+        }
+    }
+}`)
+```
+
+## New Source
+
+Specify source with data
+
+```go
+memorySource := memory.NewSource(
+	memory.WithData(data),
+)
+```
+
+## Load Source
+
+Load the source into config
+
+```go
+// Create new config
+conf := config.NewConfig()
+
+// Load file source
+conf.Load(memorySource)
+```

config/source/memory/memory.go

@@ -0,0 +1,93 @@
+// Package memory is a memory source
+package memory
+
+import (
+	"sync"
+	"time"
+
+	"github.com/micro/go-micro/config/source"
+	"github.com/pborman/uuid"
+)
+
+type memory struct {
+	sync.RWMutex
+	ChangeSet *source.ChangeSet
+	Watchers  map[string]*watcher
+}
+
+func (s *memory) Read() (*source.ChangeSet, error) {
+	s.RLock()
+	cs := &source.ChangeSet{
+		Timestamp: s.ChangeSet.Timestamp,
+		Data:      s.ChangeSet.Data,
+		Checksum:  s.ChangeSet.Checksum,
+		Source:    s.ChangeSet.Source,
+	}
+	s.RUnlock()
+	return cs, nil
+}
+
+func (s *memory) Watch() (source.Watcher, error) {
+	w := &watcher{
+		Id:      uuid.NewUUID().String(),
+		Updates: make(chan *source.ChangeSet, 100),
+		Source:  s,
+	}
+
+	s.Lock()
+	s.Watchers[w.Id] = w
+	s.Unlock()
+	return w, nil
+}
+
+// Update allows manual updates of the config data.
+func (s *memory) Update(c *source.ChangeSet) {
+	// don't process nil
+	if c == nil {
+		return
+	}
+
+	// hash the file
+	s.Lock()
+	// update changeset
+	s.ChangeSet = &source.ChangeSet{
+		Data:      c.Data,
+		Format:    c.Format,
+		Source:    "memory",
+		Timestamp: time.Now(),
+	}
+	s.ChangeSet.Checksum = s.ChangeSet.Sum()
+
+	// update watchers
+	for _, w := range s.Watchers {
+		select {
+		case w.Updates <- s.ChangeSet:
+		default:
+		}
+	}
+	s.Unlock()
+}
+
+func (s *memory) String() string {
+	return "memory"
+}
+
+func NewSource(opts ...source.Option) source.Source {
+	var options source.Options
+	for _, o := range opts {
+		o(&options)
+	}
+
+	s := &memory{
+		Watchers: make(map[string]*watcher),
+	}
+
+	if options.Context != nil {
+		c, ok := options.Context.Value(changeSetKey{}).(*source.ChangeSet)
+		if ok {
+			s.Update(c)
+		}
+	}
+
+	return s
+}

config/source/memory/options.go

@@ -0,0 +1,32 @@
+package memory
+
+import (
+	"context"
+
+	"github.com/micro/go-micro/config/source"
+)
+
+type changeSetKey struct{}
+
+// WithChangeSet allows a changeset to be set
+func WithChangeSet(cs *source.ChangeSet) source.Option {
+	return func(o *source.Options) {
+		if o.Context == nil {
+			o.Context = context.Background()
+		}
+		o.Context = context.WithValue(o.Context, changeSetKey{}, cs)
+	}
+}
+
+// WithData allows the source data to be set
+func WithData(d []byte) source.Option {
+	return func(o *source.Options) {
+		if o.Context == nil {
+			o.Context = context.Background()
+		}
+		o.Context = context.WithValue(o.Context, changeSetKey{}, &source.ChangeSet{
+			Data:   d,
+			Format: "json",
+		})
+	}
+}

config/source/memory/watcher.go

@@ -0,0 +1,23 @@
+package memory
+
+import (
+	"github.com/micro/go-micro/config/source"
+)
+
+type watcher struct {
+	Id      string
+	Updates chan *source.ChangeSet
+	Source  *memory
+}
+
+func (w *watcher) Next() (*source.ChangeSet, error) {
+	cs := <-w.Updates
+	return cs, nil
+}
+
+func (w *watcher) Stop() error {
+	w.Source.Lock()
+	delete(w.Source.Watchers, w.Id)
+	w.Source.Unlock()
+	return nil
+}

config/source/noop.go

@@ -0,0 +1,25 @@
+package source
+
+import (
+	"errors"
+)
+
+type noopWatcher struct {
+	exit chan struct{}
+}
+
+func (w *noopWatcher) Next() (*ChangeSet, error) {
+	<-w.exit
+
+	return nil, errors.New("noopWatcher stopped")
+}
+
+func (w *noopWatcher) Stop() error {
+	close(w.exit)
+	return nil
+}
+
+// NewNoopWatcher returns a watcher that blocks on Next() until Stop() is called.
+func NewNoopWatcher() (Watcher, error) {
+	return &noopWatcher{exit: make(chan struct{})}, nil
+}

config/source/options.go

@@ -0,0 +1,38 @@
+package source
+
+import (
+	"context"
+
+	"github.com/micro/go-micro/config/encoder"
+	"github.com/micro/go-micro/config/encoder/json"
+)
+
+type Options struct {
+	// Encoder
+	Encoder encoder.Encoder
+
+	// for alternative data
+	Context context.Context
+}
+
+type Option func(o *Options)
+
+func NewOptions(opts ...Option) Options {
+	options := Options{
+		Encoder: json.NewEncoder(),
+		Context: context.Background(),
+	}
+
+	for _, o := range opts {
+		o(&options)
+	}
+
+	return options
+}
+
+// WithEncoder sets the source encoder
+func WithEncoder(e encoder.Encoder) Option {
+	return func(o *Options) {
+		o.Encoder = e
+	}
+}

config/source/source.go

@@ -0,0 +1,28 @@
+// Package source is the interface for sources
+package source
+
+import (
+	"time"
+)
+
+// Source is the source from which config is loaded
+type Source interface {
+	Read() (*ChangeSet, error)
+	Watch() (Watcher, error)
+	String() string
+}
+
+// ChangeSet represents a set of changes from a source
+type ChangeSet struct {
+	Data      []byte
+	Checksum  string
+	Format    string
+	Source    string
+	Timestamp time.Time
+}
+
+// Watcher watches a source for changes
+type Watcher interface {
+	Next() (*ChangeSet, error)
+	Stop() error
+}

config/source/vault/README.md

@@ -0,0 +1,43 @@
+# Vault Source
+
+The vault source reads config from different secret engines in a Vault server. For example:
+```
+kv: secret/data/<my/secret>
+database credentials: database/creds/<my-db-role>
+```
+
+## New Source
+
+Specify source with data
+
+```go
+vaultSource := vault.NewSource(
+	// mandatory: it specifies server address. 
+	// It could have different formats:
+	// 127.0.0.1 -> https://127.0.0.1:8200
+	// http://127.0.0.1 -> http://127.0.0.1:8200
+	// http://127.0.0.1:2233
+	vault.WithAddress("http://127.0.0.1:8200"),
+	// mandatory: it specifies a resource to been access
+	vault.WithResourcePath("secret/data/my/secret"),
+    // mandatory: it specifies a resource to been access
+	vault.WithToken("<my-token>"),
+	// optional: path to store my secret.
+	// By default use resourcePath value 
+	vault.WithSecretName("my/secret"),
+	// optional: namespace.
+    vault.WithNameSpace("myNameSpace"),
+)
+```
+
+## Load Source
+
+Load the source into config
+
+```go
+// Create new config
+conf := config.NewConfig()
+
+// Load file source
+conf.Load(vaultSource)
+```

config/source/vault/options.go

@@ -0,0 +1,63 @@
+package vault
+
+import (
+	"context"
+
+	"github.com/micro/go-micro/config/source"
+)
+
+type addressKey struct{}
+type resourcePath struct{}
+type nameSpace struct{}
+type tokenKey struct{}
+type secretName struct{}
+
+// WithAddress sets the server address
+func WithAddress(a string) source.Option {
+	return func(o *source.Options) {
+		if o.Context == nil {
+			o.Context = context.Background()
+		}
+		o.Context = context.WithValue(o.Context, addressKey{}, a)
+	}
+}
+
+// WithResourcePath sets the resource that will be access
+func WithResourcePath(p string) source.Option {
+	return func(o *source.Options) {
+		if o.Context == nil {
+			o.Context = context.Background()
+		}
+		o.Context = context.WithValue(o.Context, resourcePath{}, p)
+	}
+}
+
+// WithNameSpace sets the namespace that its going to be access
+func WithNameSpace(n string) source.Option {
+	return func(o *source.Options) {
+		if o.Context == nil {
+			o.Context = context.Background()
+		}
+		o.Context = context.WithValue(o.Context, nameSpace{}, n)
+	}
+}
+
+// WithToken sets the key token to use
+func WithToken(t string) source.Option {
+	return func(o *source.Options) {
+		if o.Context == nil {
+			o.Context = context.Background()
+		}
+		o.Context = context.WithValue(o.Context, tokenKey{}, t)
+	}
+}
+
+// WithSecretName sets the name of the secret to wrap in on a map
+func WithSecretName(t string) source.Option {
+	return func(o *source.Options) {
+		if o.Context == nil {
+			o.Context = context.Background()
+		}
+		o.Context = context.WithValue(o.Context, secretName{}, t)
+	}
+}

config/source/vault/testdata/vault_init_commands

@@ -0,0 +1 @@
+vault kv put secret/data/db/auth user=myuser password=mypassword2 host=128.23.33.21 port=3307

config/source/vault/util.go

@@ -0,0 +1,98 @@
+package vault
+
+import (
+	"fmt"
+	"github.com/micro/go-micro/config/source"
+	"net"
+	"net/url"
+	"strings"
+)
+
+func makeMap(kv map[string]interface{}, secretName string) (map[string]interface{}, error) {
+	data := make(map[string]interface{})
+
+	// if secret version included
+	if kv["data"] != nil && kv["metadata"] != nil {
+		kv = kv["data"].(map[string]interface{})
+	}
+
+	target := data
+
+	// if secretName defined, wrap secrets under a map
+	if secretName != "" {
+		path := strings.Split(secretName, "/")
+		// find (or create) the location we want to put this value at
+		for i, dir := range path {
+			if _, ok := target[dir]; !ok {
+				target[dir] = make(map[string]interface{})
+			}
+			if i < len(path)-1 {
+				target = target[dir].(map[string]interface{})
+			} else {
+				target[dir] = kv
+			}
+		}
+	}
+
+	return data, nil
+}
+
+func getAddress(options source.Options) string {
+	// check if there are any addrs
+	a, ok := options.Context.Value(addressKey{}).(string)
+	if ok {
+		// check if http protocol is defined
+		if a[0] != 'h' {
+			addr, port, err := net.SplitHostPort(a)
+			if ae, ok := err.(*net.AddrError); ok && ae.Err == "missing port in address" {
+				port = "8200"
+				addr = a
+				return fmt.Sprintf("https://%s:%s", addr, port)
+			} else if err == nil {
+				return fmt.Sprintf("https://%s:%s", addr, port)
+			}
+		} else {
+			u, _ := url.Parse(a)
+
+			if host, port, _ := net.SplitHostPort(u.Host); host == "" {
+				port = "8200"
+				return fmt.Sprintf("%s://%s:%s", u.Scheme, u.Host, port)
+			} else {
+				return fmt.Sprintf("%s://%s", u.Scheme, u.Host)
+			}
+		}
+	}
+	return ""
+}
+
+func getToken(options source.Options) string {
+	token, ok := options.Context.Value(tokenKey{}).(string)
+	if ok {
+		return token
+	}
+	return ""
+}
+
+func getResourcePath(options source.Options) string {
+	path, ok := options.Context.Value(resourcePath{}).(string)
+	if ok {
+		return path
+	}
+	return ""
+}
+
+func getNameSpace(options source.Options) string {
+	ns, ok := options.Context.Value(nameSpace{}).(string)
+	if ok {
+		return ns
+	}
+	return ""
+}
+
+func getSecretName(options source.Options) string {
+	ns, ok := options.Context.Value(secretName{}).(string)
+	if ok {
+		return ns
+	}
+	return ""
+}

config/source/vault/vault.go

@@ -0,0 +1,96 @@
+package vault
+
+import (
+	"fmt"
+	"github.com/hashicorp/vault/api"
+	"github.com/micro/go-micro/config/source"
+	"time"
+)
+
+// Currently a single vault reader
+type vault struct {
+	secretPath string
+	secretName string
+	opts       source.Options
+	client     *api.Client
+}
+
+func (c *vault) Read() (*source.ChangeSet, error) {
+	secret, err := c.client.Logical().Read(c.secretPath)
+	if err != nil {
+		return nil, err
+	}
+
+	if secret == nil {
+		return nil, fmt.Errorf("source not found: %s", c.secretPath)
+	}
+
+	if secret.Data == nil && secret.Warnings != nil {
+		return nil, fmt.Errorf("source: %s errors: %v", c.secretPath, secret.Warnings)
+	}
+
+	data, err := makeMap(secret.Data, c.secretName)
+	if err != nil {
+		return nil, fmt.Errorf("error reading data: %v", err)
+	}
+
+	b, err := c.opts.Encoder.Encode(data)
+	if err != nil {
+		return nil, fmt.Errorf("error reading source: %v", err)
+	}
+
+	cs := &source.ChangeSet{
+		Timestamp: time.Now(),
+		Format:    c.opts.Encoder.String(),
+		Source:    c.String(),
+		Data:      b,
+	}
+	cs.Checksum = cs.Sum()
+
+	return cs, nil
+	//return nil, nil
+}
+
+func (c *vault) String() string {
+	return "vault"
+}
+
+func (c *vault) Watch() (source.Watcher, error) {
+	w := newWatcher(c.client)
+
+	return w, nil
+}
+
+// NewSource creates a new vault source
+func NewSource(opts ...source.Option) source.Source {
+	options := source.NewOptions(opts...)
+
+	// create the client
+	client, _ := api.NewClient(api.DefaultConfig())
+
+	// get and set options
+	if address := getAddress(options); address != "" {
+		_ = client.SetAddress(address)
+	}
+
+	if nameSpace := getNameSpace(options); nameSpace != "" {
+		client.SetNamespace(nameSpace)
+	}
+
+	if token := getToken(options); token != "" {
+		client.SetToken(token)
+	}
+
+	path := getResourcePath(options)
+	name := getSecretName(options)
+	if name == "" {
+		name = path
+	}
+
+	return &vault{
+		opts:       options,
+		client:     client,
+		secretPath: path,
+		secretName: name,
+	}
+}

config/source/vault/vault_test.go

@@ -0,0 +1,133 @@
+package vault
+
+import (
+	"encoding/json"
+	"fmt"
+	"github.com/micro/go-micro/config"
+	"os"
+	"reflect"
+	"strings"
+	"testing"
+)
+
+func TestVaultMakeMap(t *testing.T) {
+	tt := []struct {
+		name       string
+		expected   []byte
+		input      []byte
+		secretName string
+	}{
+		{
+			name:       "simple valid data 1",
+			secretName: "my/secret",
+			input:      []byte(`{"data":{"bar":"bazz", "tar":"par"}, "metadata":{"version":1, "destroyed": false}}`),
+			expected:   []byte(`{"my":{"secret":{"bar":"bazz", "tar":"par"}}}`),
+		},
+		{
+			name:       "simple valid data 2",
+			secretName: "my/secret",
+			input:      []byte(`{"bar":"bazz", "tar":"par"}`),
+			expected:   []byte(`{"my":{"secret":{"bar":"bazz", "tar":"par"}}}`),
+		},
+	}
+
+	for _, tc := range tt {
+		t.Run(tc.name, func(t *testing.T) {
+			var input map[string]interface{}
+			var expected map[string]interface{}
+
+			_ = json.Unmarshal(tc.input, &input)
+			_ = json.Unmarshal(tc.expected, &expected)
+
+			out, _ := makeMap(input, tc.secretName)
+
+			if eq := reflect.DeepEqual(out, expected); !eq {
+				fmt.Println(eq)
+				t.Fatalf("expected %v and got %v", expected, out)
+			}
+		})
+	}
+}
+
+func TestVault_Read(t *testing.T) {
+	if tr := os.Getenv("TRAVIS"); len(tr) > 0 {
+		t.Skip()
+	}
+
+	var (
+		address = "http://127.0.0.1"
+		resource = "secret/data/db/auth"
+		token = "s.Q4Zi0CSowXZl7sh0z96ijcT4"
+	)
+
+	data := []byte(`{"secret":{"data":{"db":{"auth":{"host":"128.23.33.21","password":"mypassword","port":"3306","user":"myuser"}}}}}`)
+
+	tt := []struct {
+		name      string
+		addr     string
+		resource string
+		token string
+	}{
+		{name: "read data basic", addr: address, resource: resource, token: token},
+		{name: "read data without token", addr: address, resource: resource, token: ""},
+		{name: "read data full address format", addr: "http://127.0.0.1:8200", resource: resource, token: token},
+		{name: "read data wrong resource path", addr: address, resource: "secrets/data/db/auth", token: token},
+	}
+
+	for _, tc := range tt {
+		t.Run(tc.name, func(t *testing.T) {
+			source := NewSource(
+				WithAddress(tc.addr),
+				WithResourcePath(tc.resource),
+				WithToken(tc.token),
+			)
+
+			r, err := source.Read()
+			if err != nil {
+				if tc.token == "" {
+					return
+				} else if strings.Compare(err.Error(), "source not found: secrets/data/db/auth") == 0 {
+					return
+				}
+				t.Errorf("%s: not able to read the config values because: %v", tc.name, err)
+				return
+			}
+
+			if string(r.Data) != string(data) {
+				t.Logf("data expected: %v", string(data))
+				t.Logf("data got from configmap: %v", string(r.Data))
+				t.Errorf("data from configmap does not match.")
+			}
+		})
+	}
+}
+
+func TestVault_String(t *testing.T) {
+	source := NewSource()
+
+	if source.String() != "vault" {
+		t.Errorf("expecting to get %v and instead got %v", "vault", source)
+	}
+}
+
+func TestVaultNewSource(t *testing.T) {
+	if tr := os.Getenv("TRAVIS"); len(tr) > 0 {
+		t.Skip()
+	}
+
+	conf := config.NewConfig()
+
+	_ = conf.Load(NewSource(
+		WithAddress("http://127.0.0.1"),
+		WithResourcePath("secret/data/db/auth"),
+		WithToken("s.Q4Zi0CSowXZl7sh0z96ijcT4"),
+	))
+
+	if user := conf.Get("secret", "data", "db", "auth", "user").String("user"); user != "myuser" {
+		t.Errorf("expected %v and got %v", "myuser", user)
+	}
+
+	if addr := conf.Get("secret", "data", "db", "auth", "host").String("host"); addr != "128.23.33.21" {
+		t.Errorf("expected %v and got %v", "128.23.33.21", addr)
+	}
+}

config/source/vault/watcher.go

@@ -0,0 +1,32 @@
+package vault
+
+import (
+	"errors"
+	"github.com/hashicorp/vault/api"
+	"github.com/micro/go-micro/config/source"
+)
+
+type watcher struct {
+	c    *api.Client
+	exit chan bool
+}
+
+func newWatcher(c *api.Client) *watcher {
+	return &watcher{
+		c:    c,
+		exit: make(chan bool),
+	}
+}
+
+func (w *watcher) Next() (*source.ChangeSet, error) {
+	<-w.exit
+	return nil, errors.New("url watcher stopped")
+}
+
+func (w *watcher) Stop() error {
+	select {
+	case <-w.exit:
+	default:
+	}
+	return nil
+}

config/value.go

@@ -0,0 +1,49 @@
+package config
+
+import (
+	"time"
+
+	"github.com/micro/go-micro/config/reader"
+)
+
+type value struct{}
+
+func newValue() reader.Value {
+	return new(value)
+}
+
+func (v *value) Bool(def bool) bool {
+	return false
+}
+
+func (v *value) Int(def int) int {
+	return 0
+}
+
+func (v *value) String(def string) string {
+	return ""
+}
+
+func (v *value) Float64(def float64) float64 {
+	return 0.0
+}
+
+func (v *value) Duration(def time.Duration) time.Duration {
+	return time.Duration(0)
+}
+
+func (v *value) StringSlice(def []string) []string {
+	return nil
+}
+
+func (v *value) StringMap(def map[string]string) map[string]string {
+	return map[string]string{}
+}
+
+func (v *value) Scan(val interface{}) error {
+	return nil
+}
+
+func (v *value) Bytes() []byte {
+	return nil
+}

go.mod

@@ -2,175 +2,110 @@ module github.com/micro/go-micro
 
 require (
 	cloud.google.com/go v0.39.0 // indirect
-	contrib.go.opencensus.io/exporter/ocagent v0.5.0 // indirect
+	github.com/BurntSushi/toml v0.3.1
-	github.com/Azure/azure-sdk-for-go v29.0.0+incompatible // indirect
-	github.com/Azure/go-autorest v12.1.0+incompatible // indirect
-	github.com/DataDog/dd-trace-go v1.14.0 // indirect
-	github.com/DataDog/zstd v1.4.0 // indirect
-	github.com/Jeffail/gabs v1.4.0 // indirect
-	github.com/Microsoft/go-winio v0.4.12 // indirect
-	github.com/NYTimes/gziphandler v1.1.1 // indirect
 	github.com/OneOfOne/xxhash v1.2.5 // indirect
-	github.com/PuerkitoBio/purell v1.1.1 // indirect
-	github.com/SAP/go-hdb v0.14.1 // indirect
-	github.com/Shopify/sarama v1.22.1 // indirect
-	github.com/StackExchange/wmi v0.0.0-20181212234831-e0a55b97c705 // indirect
-	github.com/aliyun/alibaba-cloud-sdk-go v0.0.0-20190522081930-582d16a078d0 // indirect
-	github.com/aliyun/aliyun-oss-go-sdk v1.9.6 // indirect
-	github.com/araddon/gou v0.0.0-20190110011759-c797efecbb61 // indirect
 	github.com/armon/go-metrics v0.0.0-20190430140413-ec5e00d3c878 // indirect
-	github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a // indirect
+	github.com/beevik/ntp v0.2.0
-	github.com/aws/aws-sdk-go v1.19.35 // indirect
+	github.com/bitly/go-simplejson v0.5.0
-	github.com/boombuler/barcode v1.0.0 // indirect
+	github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 // indirect
-	github.com/chrismalek/oktasdk-go v0.0.0-20181212195951-3430665dfaa0 // indirect
+	github.com/bradfitz/gomemcache v0.0.0-20190329173943-551aad21a668
+	github.com/bwmarrin/discordgo v0.19.0
 	github.com/containerd/continuity v0.0.0-20190426062206-aaeac12a7ffc // indirect
-	github.com/coredns/coredns v1.5.0 // indirect
+	github.com/coreos/bbolt v1.3.2 // indirect
-	github.com/coreos/etcd v3.3.13+incompatible // indirect
+	github.com/coreos/etcd v3.3.13+incompatible
 	github.com/coreos/go-semver v0.3.0 // indirect
 	github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e // indirect
-	github.com/dancannon/gorethink v4.0.0+incompatible // indirect
+	github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f // indirect
-	github.com/denisenkom/go-mssqldb v0.0.0-20190515213511-eb9f6a1743f3 // indirect
+	github.com/dgrijalva/jwt-go v3.2.0+incompatible // indirect
-	github.com/denverdino/aliyungo v0.0.0-20190410085603-611ead8a6fed // indirect
 	github.com/dgryski/go-sip13 v0.0.0-20190329191031-25c5027a8c7b // indirect
-	github.com/digitalocean/godo v1.15.0 // indirect
+	github.com/emirpasic/gods v1.12.0 // indirect
-	github.com/dnaeon/go-vcr v1.0.1 // indirect
+	github.com/forestgiant/sliceutil v0.0.0-20160425183142-94783f95db6c
-	github.com/dnstap/golang-dnstap v0.0.0-20190521061535-1a0dab85b926 // indirect
+	github.com/fsnotify/fsnotify v1.4.7
-	github.com/docker/go-connections v0.4.0 // indirect
+	github.com/fsouza/go-dockerclient v1.4.1
-	github.com/docker/go-units v0.4.0 // indirect
-	github.com/docker/spdystream v0.0.0-20181023171402-6480d4af844c // indirect
-	github.com/elazarl/go-bindata-assetfs v1.0.0 // indirect
-	github.com/elazarl/goproxy v0.0.0-20190421051319-9d40249d3c2f // indirect
-	github.com/emicklei/go-restful v2.9.5+incompatible // indirect
-	github.com/envoyproxy/go-control-plane v0.8.0 // indirect
-	github.com/evanphx/json-patch v4.2.0+incompatible // indirect
-	github.com/farsightsec/golang-framestream v0.0.0-20190425193708-fa4b164d59b8 // indirect
-	github.com/fullsailor/pkcs7 v0.0.0-20190404230743-d7302db945fa // indirect
-	github.com/gammazero/workerpool v0.0.0-20190521015540-3b91a70bc0a1 // indirect
 	github.com/garyburd/redigo v1.6.0 // indirect
-	github.com/go-ldap/ldap v3.0.3+incompatible // indirect
+	github.com/ghodss/yaml v1.0.0
+	github.com/gliderlabs/ssh v0.1.4 // indirect
 	github.com/go-log/log v0.1.0
-	github.com/go-ole/go-ole v1.2.4 // indirect
+	github.com/go-redsync/redsync v1.2.0
-	github.com/go-openapi/jsonpointer v0.19.0 // indirect
+	github.com/go-telegram-bot-api/telegram-bot-api v4.6.4+incompatible // indirect
-	github.com/go-openapi/jsonreference v0.19.0 // indirect
+	github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef // indirect
-	github.com/go-openapi/spec v0.19.0 // indirect
-	github.com/go-openapi/swag v0.19.0 // indirect
-	github.com/go-sql-driver/mysql v1.4.1 // indirect
-	github.com/go-stomp/stomp v2.0.3+incompatible // indirect
-	github.com/gocql/gocql v0.0.0-20190423091413-b99afaf3b163 // indirect
-	github.com/gogo/googleapis v1.2.0 // indirect
 	github.com/golang/mock v1.3.1 // indirect
 	github.com/golang/protobuf v1.3.1
+	github.com/gomodule/redigo v2.0.0+incompatible
 	github.com/google/btree v1.0.0 // indirect
-	github.com/google/gofuzz v1.0.0 // indirect
 	github.com/google/pprof v0.0.0-20190515194954-54271f7e092f // indirect
 	github.com/google/uuid v1.1.1
-	github.com/gophercloud/gophercloud v0.0.0-20190520235722-e87e5f90e7e6 // indirect
+	github.com/grpc-ecosystem/go-grpc-middleware v1.0.0 // indirect
-	github.com/gopherjs/gopherjs v0.0.0-20190430165422-3e4dfb77656c // indirect
+	github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect
-	github.com/gorilla/mux v1.7.2 // indirect
 	github.com/grpc-ecosystem/grpc-gateway v1.9.0 // indirect
-	github.com/hashicorp/consul v1.5.0
 	github.com/hashicorp/consul/api v1.1.0
-	github.com/hashicorp/go-checkpoint v0.5.0 // indirect
+	github.com/hashicorp/go-immutable-radix v1.1.0 // indirect
-	github.com/hashicorp/go-discover v0.0.0-20190522154730-8aba54d36e17 // indirect
-	github.com/hashicorp/go-hclog v0.9.2 // indirect
-	github.com/hashicorp/go-memdb v1.0.2 // indirect
 	github.com/hashicorp/go-msgpack v0.5.5 // indirect
-	github.com/hashicorp/go-version v1.2.0 // indirect
+	github.com/hashicorp/hcl v1.0.0
-	github.com/hashicorp/hil v0.0.0-20190212132231-97b3a9cdfa93 // indirect
+	github.com/hashicorp/mdns v1.0.1 // indirect
 	github.com/hashicorp/memberlist v0.1.4
-	github.com/hashicorp/nomad/api v0.0.0-20190522160243-df84e07c1a46 // indirect
-	github.com/hashicorp/raft v1.0.1 // indirect
-	github.com/hashicorp/raft-boltdb v0.0.0-20171010151810-6e5ba93211ea // indirect
 	github.com/hashicorp/serf v0.8.3 // indirect
-	github.com/hashicorp/vault v1.1.2 // indirect
+	github.com/hashicorp/vault/api v1.0.2
-	github.com/hashicorp/vault-plugin-auth-alicloud v0.5.1 // indirect
+	github.com/imdario/mergo v0.3.7
-	github.com/hashicorp/vault-plugin-auth-azure v0.5.1 // indirect
+	github.com/jonboulle/clockwork v0.1.0 // indirect
-	github.com/hashicorp/vault-plugin-auth-centrify v0.5.1 // indirect
+	github.com/json-iterator/go v1.1.6 // indirect
-	github.com/hashicorp/vault-plugin-auth-jwt v0.5.1 // indirect
-	github.com/hashicorp/vault-plugin-auth-kubernetes v0.5.1 // indirect
-	github.com/hashicorp/vault-plugin-secrets-ad v0.5.1 // indirect
-	github.com/hashicorp/vault-plugin-secrets-alicloud v0.5.1 // indirect
-	github.com/hashicorp/vault-plugin-secrets-azure v0.5.1 // indirect
-	github.com/hashicorp/vault-plugin-secrets-gcp v0.5.2 // indirect
-	github.com/hashicorp/vault-plugin-secrets-gcpkms v0.5.1 // indirect
-	github.com/hashicorp/vault-plugin-secrets-kv v0.5.1 // indirect
-	github.com/hashicorp/vault/api v1.0.2 // indirect
-	github.com/hashicorp/vault/sdk v0.1.11 // indirect
-	github.com/influxdata/influxdb v1.7.6 // indirect
-	github.com/jarcoal/httpmock v1.0.4 // indirect
-	github.com/jefferai/jsonx v1.0.0 // indirect
-	github.com/joyent/triton-go v0.0.0-20190112182421-51ffac552869 // indirect
-	github.com/keybase/go-crypto v0.0.0-20190416182011-b785b22cc757 // indirect
 	github.com/kisielk/errcheck v1.2.0 // indirect
-	github.com/klauspost/cpuid v1.2.1 // indirect
 	github.com/konsorten/go-windows-terminal-sequences v1.0.2 // indirect
 	github.com/kr/pty v1.1.4 // indirect
-	github.com/kylelemons/godebug v1.1.0 // indirect
+	github.com/lusis/go-slackbot v0.0.0-20180109053408-401027ccfef5 // indirect
-	github.com/lib/pq v1.1.1 // indirect
+	github.com/lusis/slack-test v0.0.0-20190426140909-c40012f20018 // indirect
-	github.com/linode/linodego v0.8.1 // indirect
+	github.com/mattn/go-colorable v0.1.2 // indirect
-	github.com/lucas-clemente/quic-go v0.11.1 // indirect
+	github.com/micro/cli v0.2.0
-	github.com/lyft/protoc-gen-validate v0.0.14 // indirect
-	github.com/mailru/easyjson v0.0.0-20190403194419-1ea4449da983 // indirect
-	github.com/mattbaird/elastigo v0.0.0-20170123220020-2fe47fd29e4b // indirect
-	github.com/mattn/go-isatty v0.0.8 // indirect
-	github.com/mholt/caddy v1.0.0 // indirect
-	github.com/mholt/certmagic v0.5.1 // indirect
-	github.com/michaelklishin/rabbit-hole v1.5.0 // indirect
-	github.com/micro/cli v0.1.0
 	github.com/micro/go-log v0.1.0
 	github.com/micro/go-rcache v0.3.0
 	github.com/micro/mdns v0.1.0
-	github.com/micro/util v0.2.0
+	github.com/miekg/dns v1.1.13 // indirect
-	github.com/miekg/dns v1.1.12 // indirect
 	github.com/mitchellh/hashstructure v1.0.0
-	github.com/mitchellh/pointerstructure v0.0.0-20190430161007-f252a8fd71c8 // indirect
+	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
-	github.com/munnerz/goautoneg v0.0.0-20190414153302-2ae31c8b6b30 // indirect
+	github.com/modern-go/reflect2 v1.0.1 // indirect
-	github.com/opentracing/opentracing-go v1.1.0 // indirect
+	github.com/nlopes/slack v0.5.0
-	github.com/openzipkin/zipkin-go-opentracing v0.3.5 // indirect
+	github.com/onsi/ginkgo v1.8.0 // indirect
-	github.com/ory-am/common v0.4.0 // indirect
+	github.com/onsi/gomega v1.5.0 // indirect
-	github.com/pborman/uuid v1.2.0 // indirect
+	github.com/pborman/uuid v1.2.0
 	github.com/pkg/errors v0.8.1
-	github.com/pquerna/otp v1.1.0 // indirect
 	github.com/prometheus/client_golang v0.9.3 // indirect
 	github.com/prometheus/common v0.4.1 // indirect
-	github.com/prometheus/procfs v0.0.0-20190522114515-bc1a522cf7b1 // indirect
+	github.com/prometheus/procfs v0.0.1 // indirect
 	github.com/prometheus/tsdb v0.8.0 // indirect
 	github.com/rogpeppe/fastuuid v1.1.0 // indirect
-	github.com/russross/blackfriday v2.0.0+incompatible // indirect
-	github.com/samuel/go-zookeeper v0.0.0-20180130194729-c4fab1ac1bec // indirect
-	github.com/shirou/gopsutil v2.18.12+incompatible // indirect
 	github.com/sirupsen/logrus v1.4.2 // indirect
-	github.com/smartystreets/assertions v0.0.0-20190401211740-f487f9de1cd3 // indirect
+	github.com/soheilhy/cmux v0.1.4 // indirect
-	github.com/softlayer/softlayer-go v0.0.0-20190508182157-7c592eb2559c // indirect
 	github.com/spaolacci/murmur3 v1.1.0 // indirect
-	github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94 // indirect
 	github.com/stretchr/objx v0.2.0 // indirect
-	github.com/ugorji/go v1.1.4 // indirect
+	github.com/technoweenie/multipartstreamer v1.0.1 // indirect
-	github.com/vmware/govmomi v0.20.1 // indirect
+	github.com/tmc/grpc-websocket-proxy v0.0.0-20190109142713-0ad062ec5ee5 // indirect
+	github.com/xanzy/ssh-agent v0.2.1 // indirect
+	github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect
+	go.etcd.io/bbolt v1.3.2 // indirect
+	go.etcd.io/etcd v3.3.13+incompatible
+	go.opencensus.io v0.22.0 // indirect
 	go.uber.org/atomic v1.4.0 // indirect
+	go.uber.org/multierr v1.1.0 // indirect
 	go.uber.org/zap v1.10.0 // indirect
-	golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f // indirect
+	golang.org/x/crypto v0.0.0-20190530122614-20be4c3c3ed5 // indirect
 	golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522 // indirect
-	golang.org/x/image v0.0.0-20190516052701-61b8692d9a5c // indirect
+	golang.org/x/image v0.0.0-20190523035834-f03afa92d3ff // indirect
 	golang.org/x/lint v0.0.0-20190409202823-959b441ac422 // indirect
 	golang.org/x/mobile v0.0.0-20190509164839-32b2708ab171 // indirect
 	golang.org/x/net v0.0.0-20190522155817-f3200d17e092
-	golang.org/x/oauth2 v0.0.0-20190517181255-950ef44c6e07 // indirect
+	golang.org/x/oauth2 v0.0.0-20190523182746-aaccbc9213b0 // indirect
-	golang.org/x/sys v0.0.0-20190522044717-8097e1b27ff5 // indirect
+	golang.org/x/sys v0.0.0-20190531073156-46560c3f3c0a // indirect
-	golang.org/x/tools v0.0.0-20190521203540-521d6ed310dd // indirect
+	golang.org/x/tools v0.0.0-20190530215528-75312fb06703 // indirect
 	google.golang.org/appengine v1.6.0 // indirect
-	google.golang.org/genproto v0.0.0-20190516172635-bb713bdc0e52 // indirect
+	google.golang.org/genproto v0.0.0-20190530194941-fb225487d101 // indirect
-	gopkg.in/gorethink/gorethink.v4 v4.1.0 // indirect
+	google.golang.org/grpc v1.21.0 // indirect
-	gopkg.in/mgo.v2 v2.0.0-20180705113604-9856a29383ce // indirect
+	gopkg.in/bsm/ratelimit.v1 v1.0.0-20160220154919-db14e161995a // indirect
-	gopkg.in/ory-am/dockertest.v2 v2.2.3 // indirect
+	gopkg.in/redis.v3 v3.6.4
-	honnef.co/go/tools v0.0.0-20190522022531-bad1bd262ba8 // indirect
+	gopkg.in/src-d/go-billy.v4 v4.3.0 // indirect
-	k8s.io/api v0.0.0-20190515023547-db5a9d1c40eb // indirect
+	gopkg.in/src-d/go-git-fixtures.v3 v3.5.0 // indirect
-	k8s.io/client-go v11.0.0+incompatible // indirect
+	gopkg.in/src-d/go-git.v4 v4.11.0
-	k8s.io/gengo v0.0.0-20190327210449-e17681d19d3a // indirect
+	gopkg.in/telegram-bot-api.v4 v4.6.4
-	k8s.io/kube-openapi v0.0.0-20190510232812-a01b7d5d6c22 // indirect
+	honnef.co/go/tools v0.0.0-20190530170028-a1efa522b896 // indirect
-	k8s.io/utils v0.0.0-20190520173318-324c5df7d3f0 // indirect
-	layeh.com/radius v0.0.0-20190322222518-890bc1058917 // indirect
-	sigs.k8s.io/structured-merge-diff v0.0.0-20190521201008-1c46bef2e9c8 // indirect
 )
 
 replace github.com/golang/lint => github.com/golang/lint v0.0.0-20190227174305-8f45f776aaf1

registry/cache/README.md

@@ -0,0 +1,31 @@
+# Registry Cache 
+
+Cache is a library that provides a caching layer for the go-micro [registry](https://godoc.org/github.com/micro/go-micro/registry#Registry).
+
+If you're looking for caching in your microservices use the [selector](https://micro.mu/docs/fault-tolerance.html#caching-discovery).
+
+## Interface
+
+```
+// Cache is the registry cache interface
+type Cache interface {
+	// embed the registry interface
+	registry.Registry
+	// stop the cache watcher
+	Stop()
+}
+```
+
+## Usage
+
+```
+import (
+	"github.com/micro/go-micro/registry"
+	"github.com/micro/go-micro/registry/cache"
+)
+
+r := registry.NewRegistry()
+cache := cache.New(r)
+
+services, _ := cache.GetService("my.service")
+```

registry/cache/options.go

@@ -0,0 +1,12 @@
+package cache
+
+import (
+	"time"
+)
+
+// WithTTL sets the cache TTL
+func WithTTL(t time.Duration) Option {
+	return func(o *Options) {
+		o.TTL = t
+	}
+}

registry/cache/rcache.go

@@ -0,0 +1,429 @@
+// Package cache provides a registry cache
+package cache
+
+import (
+	"math"
+	"math/rand"
+	"sync"
+	"time"
+
+	"github.com/micro/go-micro/registry"
+	log "github.com/micro/go-micro/util/log"
+)
+
+// Cache is the registry cache interface
+type Cache interface {
+	// embed the registry interface
+	registry.Registry
+	// stop the cache watcher
+	Stop()
+}
+
+type Options struct {
+	// TTL is the cache TTL
+	TTL time.Duration
+}
+
+type Option func(o *Options)
+
+type cache struct {
+	registry.Registry
+	opts Options
+
+	// registry cache
+	sync.RWMutex
+	cache   map[string][]*registry.Service
+	ttls    map[string]time.Time
+	watched map[string]bool
+
+	exit chan bool
+}
+
+var (
+	DefaultTTL = time.Minute
+)
+
+func backoff(attempts int) time.Duration {
+	if attempts == 0 {
+		return time.Duration(0)
+	}
+	return time.Duration(math.Pow(10, float64(attempts))) * time.Millisecond
+}
+
+// isValid checks if the service is valid
+func (c *cache) isValid(services []*registry.Service, ttl time.Time) bool {
+	// no services exist
+	if len(services) == 0 {
+		return false
+	}
+
+	// ttl is invalid
+	if ttl.IsZero() {
+		return false
+	}
+
+	// time since ttl is longer than timeout
+	if time.Since(ttl) > c.opts.TTL {
+		return false
+	}
+
+	// ok
+	return true
+}
+
+func (c *cache) quit() bool {
+	select {
+	case <-c.exit:
+		return true
+	default:
+		return false
+	}
+}
+
+// cp copies a service. Because we're caching handing back pointers would
+// create a race condition, so we do this instead its fast enough
+func (c *cache) cp(current []*registry.Service) []*registry.Service {
+	var services []*registry.Service
+
+	for _, service := range current {
+		// copy service
+		s := new(registry.Service)
+		*s = *service
+
+		// copy nodes
+		var nodes []*registry.Node
+		for _, node := range service.Nodes {
+			n := new(registry.Node)
+			*n = *node
+			nodes = append(nodes, n)
+		}
+		s.Nodes = nodes
+
+		// copy endpoints
+		var eps []*registry.Endpoint
+		for _, ep := range service.Endpoints {
+			e := new(registry.Endpoint)
+			*e = *ep
+			eps = append(eps, e)
+		}
+		s.Endpoints = eps
+
+		// append service
+		services = append(services, s)
+	}
+
+	return services
+}
+
+func (c *cache) del(service string) {
+	delete(c.cache, service)
+	delete(c.ttls, service)
+}
+
+func (c *cache) get(service string) ([]*registry.Service, error) {
+	// read lock
+	c.RLock()
+
+	// check the cache first
+	services := c.cache[service]
+	// get cache ttl
+	ttl := c.ttls[service]
+
+	// got services && within ttl so return cache
+	if c.isValid(services, ttl) {
+		// make a copy
+		cp := c.cp(services)
+		// unlock the read
+		c.RUnlock()
+		// return servics
+		return cp, nil
+	}
+
+	// get does the actual request for a service and cache it
+	get := func(service string) ([]*registry.Service, error) {
+		// ask the registry
+		services, err := c.Registry.GetService(service)
+		if err != nil {
+			return nil, err
+		}
+
+		// cache results
+		c.Lock()
+		c.set(service, c.cp(services))
+		c.Unlock()
+
+		return services, nil
+	}
+
+	// watch service if not watched
+	if _, ok := c.watched[service]; !ok {
+		go c.run(service)
+	}
+
+	// unlock the read lock
+	c.RUnlock()
+
+	// get and return services
+	return get(service)
+}
+
+func (c *cache) set(service string, services []*registry.Service) {
+	c.cache[service] = services
+	c.ttls[service] = time.Now().Add(c.opts.TTL)
+}
+
+func (c *cache) update(res *registry.Result) {
+	if res == nil || res.Service == nil {
+		return
+	}
+
+	c.Lock()
+	defer c.Unlock()
+
+	services, ok := c.cache[res.Service.Name]
+	if !ok {
+		// we're not going to cache anything
+		// unless there was already a lookup
+		return
+	}
+
+	if len(res.Service.Nodes) == 0 {
+		switch res.Action {
+		case "delete":
+			c.del(res.Service.Name)
+		}
+		return
+	}
+
+	// existing service found
+	var service *registry.Service
+	var index int
+	for i, s := range services {
+		if s.Version == res.Service.Version {
+			service = s
+			index = i
+		}
+	}
+
+	switch res.Action {
+	case "create", "update":
+		if service == nil {
+			c.set(res.Service.Name, append(services, res.Service))
+			return
+		}
+
+		// append old nodes to new service
+		for _, cur := range service.Nodes {
+			var seen bool
+			for _, node := range res.Service.Nodes {
+				if cur.Id == node.Id {
+					seen = true
+					break
+				}
+			}
+			if !seen {
+				res.Service.Nodes = append(res.Service.Nodes, cur)
+			}
+		}
+
+		services[index] = res.Service
+		c.set(res.Service.Name, services)
+	case "delete":
+		if service == nil {
+			return
+		}
+
+		var nodes []*registry.Node
+
+		// filter cur nodes to remove the dead one
+		for _, cur := range service.Nodes {
+			var seen bool
+			for _, del := range res.Service.Nodes {
+				if del.Id == cur.Id {
+					seen = true
+					break
+				}
+			}
+			if !seen {
+				nodes = append(nodes, cur)
+			}
+		}
+
+		// still got nodes, save and return
+		if len(nodes) > 0 {
+			service.Nodes = nodes
+			services[index] = service
+			c.set(service.Name, services)
+			return
+		}
+
+		// zero nodes left
+
+		// only have one thing to delete
+		// nuke the thing
+		if len(services) == 1 {
+			c.del(service.Name)
+			return
+		}
+
+		// still have more than 1 service
+		// check the version and keep what we know
+		var srvs []*registry.Service
+		for _, s := range services {
+			if s.Version != service.Version {
+				srvs = append(srvs, s)
+			}
+		}
+
+		// save
+		c.set(service.Name, srvs)
+	}
+}
+
+// run starts the cache watcher loop
+// it creates a new watcher if there's a problem
+func (c *cache) run(service string) {
+	// set watcher
+	c.Lock()
+	c.watched[service] = true
+	c.Unlock()
+
+	// delete watcher on exit
+	defer func() {
+		c.Lock()
+		delete(c.watched, service)
+		c.Unlock()
+	}()
+
+	var a, b int
+
+	for {
+		// exit early if already dead
+		if c.quit() {
+			return
+		}
+
+		// jitter before starting
+		j := rand.Int63n(100)
+		time.Sleep(time.Duration(j) * time.Millisecond)
+
+		// create new watcher
+		w, err := c.Registry.Watch(
+			registry.WatchService(service),
+		)
+
+		if err != nil {
+			if c.quit() {
+				return
+			}
+
+			d := backoff(a)
+
+			if a > 3 {
+				log.Log("rcache: ", err, " backing off ", d)
+				a = 0
+			}
+
+			time.Sleep(d)
+			a++
+
+			continue
+		}
+
+		// reset a
+		a = 0
+
+		// watch for events
+		if err := c.watch(w); err != nil {
+			if c.quit() {
+				return
+			}
+
+			d := backoff(b)
+
+			if b > 3 {
+				log.Log("rcache: ", err, " backing off ", d)
+				b = 0
+			}
+
+			time.Sleep(d)
+			b++
+
+			continue
+		}
+
+		// reset b
+		b = 0
+	}
+}
+
+// watch loops the next event and calls update
+// it returns if there's an error
+func (c *cache) watch(w registry.Watcher) error {
+	defer w.Stop()
+
+	// manage this loop
+	go func() {
+		// wait for exit
+		<-c.exit
+		w.Stop()
+	}()
+
+	for {
+		res, err := w.Next()
+		if err != nil {
+			return err
+		}
+		c.update(res)
+	}
+}
+
+func (c *cache) GetService(service string) ([]*registry.Service, error) {
+	// get the service
+	services, err := c.get(service)
+	if err != nil {
+		return nil, err
+	}
+
+	// if there's nothing return err
+	if len(services) == 0 {
+		return nil, registry.ErrNotFound
+	}
+
+	// return services
+	return services, nil
+}
+
+func (c *cache) Stop() {
+	select {
+	case <-c.exit:
+		return
+	default:
+		close(c.exit)
+	}
+}
+
+func (c *cache) String() string {
+	return "rcache"
+}
+
+// New returns a new cache
+func New(r registry.Registry, opts ...Option) Cache {
+	rand.Seed(time.Now().UnixNano())
+	options := Options{
+		TTL: DefaultTTL,
+	}
+
+	for _, o := range opts {
+		o(&options)
+	}
+
+	return &cache{
+		Registry: r,
+		opts:     options,
+		watched:  make(map[string]bool),
+		cache:    make(map[string][]*registry.Service),
+		ttls:     make(map[string]time.Time),
+		exit:     make(chan bool),
+	}
+}

registry/gossip/gossip.go

@@ -16,9 +16,9 @@ import (
 	"github.com/golang/protobuf/proto"
 	"github.com/google/uuid"
 	"github.com/hashicorp/memberlist"
-	log "github.com/micro/go-log"
 	"github.com/micro/go-micro/registry"
 	pb "github.com/micro/go-micro/registry/gossip/proto"
+	log "github.com/micro/go-micro/util/log"
 	"github.com/mitchellh/hashstructure"
 )
 

runtime/.travis.yml

@@ -0,0 +1,7 @@
+language: go
+go:
+- 1.9.4
+- 1.10.x
+notifications:
+  slack:
+    secure: fQJsxnChA2JDbdNti8cLiBlVj/ws38vaLyDJzkM+jhrU8NmdPbnjYu99lauDak0Yy9qdLNYUUO1KP6Kn8gl5VcbdC0++T3jPNCgIkJpvm/DJZKKtWlDXCavCDZ3EtCv8IhviD2dguHGtcX08/G6zc9V44GQu3sPWDdGdYnplh7OBIvicJZ5v8o6TR/WZ/4waGCLxeR84fIamXcZTlHGk3hJMKqCsb6urJX42rOlfSgjourb1yu8Tyg8uTXsO3Z7fgiZ73mYvZ1iD1v5thHB8T6bfB4g3U3KE53IjV9oV9bL6iDoiHbPWTIHD6yaZKXDbMxyw/sZ70GgxXH795pJeVBAWv9X6nH+aemwR72kgszvDLq64jxeAqJAmK5OqRs0YSQMKmvceSCIbEptCnpdl0KhhDZg/ojsfJOHyJb/SrEcntuvvig7O3Qyp9OtJdXJa5JnXi7Z2tPDXy+QnL/WSXUHU7JnDfQeQJC6QawxVGllxECcStIVb2/FrT0wnEinS66uhQhW48zLFcbJIFYNVsTlCMkTjXBkSvbjn3qSQr9IaMvpmDXhp9EC99Y5DgkPFP0S7aGjPCxfgrMHxqyxeEwmJQ+vYO6sZxnZbOzUMYVgPdAptQJ30QjovoF3lcQoBnZjvoKdunKE4qvSFYsMlxDV0SJhS5bc+eIXNIN5jYE8=

runtime/README.md

@@ -0,0 +1,28 @@
+# Runtime
+
+A runtime for self governing services.
+
+## Overview
+
+In recent years we've started to develop complex architectures for the pipeline between writing code and running it. This 
+philosophy of build, run, manage or however many variations, has created a number of layers of abstraction that make it 
+all the more difficult to run code.
+
+Runtime manages the lifecycle of a service from source to running process. If the source is the *source of truth* then 
+everything in between running is wasted breath. Applications should be self governing and self sustaining. 
+To enable that we need libraries which make it possible.
+
+Runtime will fetch source code, build a binary and execute it. Any Go program that uses this library should be able 
+to run dependencies or itself with ease, with the ability to update itself as the source is updated.
+
+## Features
+
+- **Source** - Fetches source whether it be git, go, docker, etc
+- **Package** - Compiles the source into a binary which can be executed
+- **Process** - Executes a binary and creates a running process
+
+## Usage
+
+TODO
+
+

runtime/package/docker/docker.go

@@ -0,0 +1,93 @@
+// Package docker builds docker images
+package docker
+
+import (
+	"archive/tar"
+	"bytes"
+	"io/ioutil"
+	"os"
+	"path/filepath"
+
+	"github.com/fsouza/go-dockerclient"
+	"github.com/micro/go-micro/runtime/package"
+	"github.com/micro/go-micro/util/log"
+)
+
+type Packager struct {
+	Options packager.Options
+	Client  *docker.Client
+}
+
+func (d *Packager) Compile(s *packager.Source) (*packager.Binary, error) {
+	image := filepath.Join(s.Repository.Path, s.Repository.Name)
+
+	buf := new(bytes.Buffer)
+	tw := tar.NewWriter(buf)
+	defer tw.Close()
+
+	dockerFile := "Dockerfile"
+
+	// open docker file
+	f, err := os.Open(filepath.Join(s.Repository.Path, s.Repository.Name, dockerFile))
+	if err != nil {
+		return nil, err
+	}
+	// read docker file
+	by, err := ioutil.ReadAll(f)
+	if err != nil {
+		return nil, err
+	}
+
+	tarHeader := &tar.Header{
+		Name: dockerFile,
+		Size: int64(len(by)),
+	}
+	err = tw.WriteHeader(tarHeader)
+	if err != nil {
+		return nil, err
+	}
+	_, err = tw.Write(by)
+	if err != nil {
+		return nil, err
+	}
+	tr := bytes.NewReader(buf.Bytes())
+
+	err = d.Client.BuildImage(docker.BuildImageOptions{
+		Name:           image,
+		Dockerfile:     dockerFile,
+		InputStream:    tr,
+		OutputStream:   ioutil.Discard,
+		RmTmpContainer: true,
+		SuppressOutput: true,
+	})
+	if err != nil {
+		return nil, err
+	}
+	return &packager.Binary{
+		Name:   image,
+		Path:   image,
+		Type:   "docker",
+		Source: s,
+	}, nil
+}
+
+func (d *Packager) Delete(b *packager.Binary) error {
+	image := filepath.Join(b.Path, b.Name)
+	return d.Client.RemoveImage(image)
+}
+
+func NewPackager(opts ...packager.Option) packager.Packager {
+	options := packager.Options{}
+	for _, o := range opts {
+		o(&options)
+	}
+	endpoint := "unix:///var/run/docker.sock"
+	client, err := docker.NewClient(endpoint)
+	if err != nil {
+		log.Fatal(err)
+	}
+	return &Packager{
+		Options: options,
+		Client:  client,
+	}
+}