add config

config/source/changeset.go

@@ -0,0 +1,13 @@
+package source
+
+import (
+	"crypto/md5"
+	"fmt"
+)
+
+// Sum returns the md5 checksum of the ChangeSet data
+func (c *ChangeSet) Sum() string {
+	h := md5.New()
+	h.Write(c.Data)
+	return fmt.Sprintf("%x", h.Sum(nil))
+}

config/source/cli/README.md

@@ -0,0 +1,71 @@
+# cli Source
+
+The cli source reads config from parsed flags via a cli.Context.
+
+## Format
+
+We expect the use of the `micro/cli` package. Upper case flags will be lower cased. Dashes will be used as delimiters for nesting.
+
+### Example
+
+```go
+micro.Flags(
+    cli.StringFlag{
+        Name: "database-address",
+        Value: "127.0.0.1",
+        Usage: "the db address",
+    },
+    cli.IntFlag{
+        Name: "database-port",
+        Value: 3306,
+        Usage: "the db port",
+    },
+)
+```
+
+Becomes
+
+```json
+{
+    "database": {
+        "address": "127.0.0.1",
+        "port": 3306
+    }
+}
+```
+
+## New and Load Source
+
+Because a cli.Context is needed to retrieve the flags and their values, it is recommended to build your source from within a cli.Action.
+
+```go
+
+func main() {
+    // New Service
+    service := micro.NewService(
+        micro.Name("example"),
+        micro.Flags(
+            cli.StringFlag{
+                Name: "database-address",
+                Value: "127.0.0.1",
+                Usage: "the db address",
+            },
+        ),
+    )
+
+    var clisrc source.Source
+
+    service.Init(
+        micro.Action(func(c *cli.Context) {
+            clisrc = cli.NewSource(
+                cli.Context(c),
+	    )
+            // Alternatively, just setup your config right here
+        }),
+    )
+    
+    // ... Load and use that source ...
+    conf := config.NewConfig()
+    conf.Load(clisrc)
+}
+```

config/source/cli/cli.go

@@ -0,0 +1,146 @@
+package cli
+
+import (
+	"flag"
+	"io/ioutil"
+	"os"
+	"strings"
+	"time"
+
+	"github.com/imdario/mergo"
+	"github.com/micro/cli"
+	"github.com/micro/go-micro/cmd"
+	"github.com/micro/go-micro/config/source"
+)
+
+type cliSource struct {
+	opts source.Options
+	ctx  *cli.Context
+}
+
+func (c *cliSource) Read() (*source.ChangeSet, error) {
+	var changes map[string]interface{}
+
+	for _, name := range c.ctx.GlobalFlagNames() {
+		tmp := toEntry(name, c.ctx.GlobalGeneric(name))
+		mergo.Map(&changes, tmp) // need to sort error handling
+	}
+
+	for _, name := range c.ctx.FlagNames() {
+		tmp := toEntry(name, c.ctx.Generic(name))
+		mergo.Map(&changes, tmp) // need to sort error handling
+	}
+
+	b, err := c.opts.Encoder.Encode(changes)
+	if err != nil {
+		return nil, err
+	}
+
+	cs := &source.ChangeSet{
+		Format:    c.opts.Encoder.String(),
+		Data:      b,
+		Timestamp: time.Now(),
+		Source:    c.String(),
+	}
+	cs.Checksum = cs.Sum()
+
+	return cs, nil
+}
+
+func toEntry(name string, v interface{}) map[string]interface{} {
+	n := strings.ToLower(name)
+	keys := strings.FieldsFunc(n, split)
+	reverse(keys)
+	tmp := make(map[string]interface{})
+	for i, k := range keys {
+		if i == 0 {
+			tmp[k] = v
+			continue
+		}
+
+		tmp = map[string]interface{}{k: tmp}
+	}
+	return tmp
+}
+
+func reverse(ss []string) {
+	for i := len(ss)/2 - 1; i >= 0; i-- {
+		opp := len(ss) - 1 - i
+		ss[i], ss[opp] = ss[opp], ss[i]
+	}
+}
+
+func split(r rune) bool {
+	return r == '-' || r == '_'
+}
+
+func (c *cliSource) Watch() (source.Watcher, error) {
+	return source.NewNoopWatcher()
+}
+
+func (c *cliSource) String() string {
+	return "cli"
+}
+
+// NewSource returns a config source for integrating parsed flags from a micro/cli.Context.
+// Hyphens are delimiters for nesting, and all keys are lowercased. The assumption is that
+// command line flags have already been parsed.
+//
+// Example:
+//      cli.StringFlag{Name: "db-host"},
+//
+//
+//      {
+//          "database": {
+//              "host": "localhost"
+//          }
+//      }
+func NewSource(opts ...source.Option) source.Source {
+	options := source.NewOptions(opts...)
+
+	var ctx *cli.Context
+
+	c, ok := options.Context.Value(contextKey{}).(*cli.Context)
+	if ok {
+		ctx = c
+	}
+
+	// no context
+	if ctx == nil {
+		// get the default app/flags
+		app := cmd.App()
+		flags := app.Flags
+
+		// create flagset
+		set := flag.NewFlagSet(app.Name, flag.ContinueOnError)
+
+		// apply flags to set
+		for _, f := range flags {
+			f.Apply(set)
+		}
+
+		// parse flags
+		set.SetOutput(ioutil.Discard)
+		set.Parse(os.Args[1:])
+
+		// normalise flags
+		normalizeFlags(app.Flags, set)
+
+		// create context
+		ctx = cli.NewContext(app, set, nil)
+	}
+
+	return &cliSource{
+		ctx:  ctx,
+		opts: options,
+	}
+}
+
+// WithContext returns a new source with the context specified.
+// The assumption is that Context is retrieved within an app.Action function.
+func WithContext(ctx *cli.Context, opts ...source.Option) source.Source {
+	return &cliSource{
+		ctx:  ctx,
+		opts: source.NewOptions(opts...),
+	}
+}

config/source/cli/cli_test.go

@@ -0,0 +1,65 @@
+package cli
+
+import (
+	"encoding/json"
+	"os"
+	"testing"
+
+	"github.com/micro/cli"
+	"github.com/micro/go-micro/cmd"
+	"github.com/micro/go-micro/config/source"
+)
+
+func test(t *testing.T, withContext bool) {
+	var src source.Source
+
+	// setup app
+	app := cmd.App()
+	app.Name = "testapp"
+	app.Flags = []cli.Flag{
+		cli.StringFlag{Name: "db-host"},
+	}
+
+	// with context
+	if withContext {
+		// set action
+		app.Action = func(c *cli.Context) {
+			src = WithContext(c)
+		}
+
+		// run app
+		app.Run([]string{"run", "-db-host", "localhost"})
+		// no context
+	} else {
+		// set args
+		os.Args = []string{"run", "-db-host", "localhost"}
+		src = NewSource()
+	}
+
+	// test config
+	c, err := src.Read()
+	if err != nil {
+		t.Error(err)
+	}
+
+	var actual map[string]interface{}
+	if err := json.Unmarshal(c.Data, &actual); err != nil {
+		t.Error(err)
+	}
+
+	actualDB := actual["db"].(map[string]interface{})
+	if actualDB["host"] != "localhost" {
+		t.Errorf("expected localhost, got %v", actualDB["name"])
+	}
+
+}
+
+func TestCliSource(t *testing.T) {
+	// without context
+	test(t, false)
+}
+
+func TestCliSourceWithContext(t *testing.T) {
+	// with context
+	test(t, true)
+}

config/source/cli/options.go

@@ -0,0 +1,20 @@
+package cli
+
+import (
+	"context"
+
+	"github.com/micro/cli"
+	"github.com/micro/go-micro/config/source"
+)
+
+type contextKey struct{}
+
+// Context sets the cli context
+func Context(c *cli.Context) source.Option {
+	return func(o *source.Options) {
+		if o.Context == nil {
+			o.Context = context.Background()
+		}
+		o.Context = context.WithValue(o.Context, contextKey{}, c)
+	}
+}

config/source/cli/util.go

@@ -0,0 +1,50 @@
+package cli
+
+import (
+	"errors"
+	"flag"
+	"strings"
+
+	"github.com/micro/cli"
+)
+
+func copyFlag(name string, ff *flag.Flag, set *flag.FlagSet) {
+	switch ff.Value.(type) {
+	case *cli.StringSlice:
+	default:
+		set.Set(name, ff.Value.String())
+	}
+}
+
+func normalizeFlags(flags []cli.Flag, set *flag.FlagSet) error {
+	visited := make(map[string]bool)
+	set.Visit(func(f *flag.Flag) {
+		visited[f.Name] = true
+	})
+	for _, f := range flags {
+		parts := strings.Split(f.GetName(), ",")
+		if len(parts) == 1 {
+			continue
+		}
+		var ff *flag.Flag
+		for _, name := range parts {
+			name = strings.Trim(name, " ")
+			if visited[name] {
+				if ff != nil {
+					return errors.New("Cannot use two forms of the same flag: " + name + " " + ff.Name)
+				}
+				ff = set.Lookup(name)
+			}
+		}
+		if ff == nil {
+			continue
+		}
+		for _, name := range parts {
+			name = strings.Trim(name, " ")
+			if !visited[name] {
+				copyFlag(name, ff, set)
+			}
+		}
+	}
+	return nil
+}

config/source/consul/README.md

@@ -0,0 +1,49 @@
+# Consul Source
+
+The consul source reads config from consul key/values
+
+## Consul Format
+
+The consul source expects keys under the default prefix `/micro/config`
+
+Values are expected to be json
+
+```
+// set database
+consul kv put micro/config/database '{"address": "10.0.0.1", "port": 3306}'
+// set cache
+consul kv put micro/config/cache '{"address": "10.0.0.2", "port": 6379}'
+```
+
+Keys are split on `/` so access becomes
+
+```
+conf.Get("micro", "config", "database")
+```
+
+## New Source
+
+Specify source with data
+
+```go
+consulSource := consul.NewSource(
+	// optionally specify consul address; default to localhost:8500
+	consul.WithAddress("10.0.0.10:8500"),
+	// optionally specify prefix; defaults to /micro/config
+	consul.WithPrefix("/my/prefix"),
+  // optionally strip the provided prefix from the keys, defaults to false
+  consul.StripPrefix(true),
+)
+```
+
+## Load Source
+
+Load the source into config
+
+```go
+// Create new config
+conf := config.NewConfig()
+
+// Load file source
+conf.Load(consulSource)
+```

config/source/consul/consul.go

@@ -0,0 +1,121 @@
+package consul
+
+import (
+	"fmt"
+	"net"
+	"time"
+
+	"github.com/hashicorp/consul/api"
+	"github.com/micro/go-micro/config/source"
+)
+
+// Currently a single consul reader
+type consul struct {
+	prefix      string
+	stripPrefix string
+	addr        string
+	opts        source.Options
+	client      *api.Client
+}
+
+var (
+	// DefaultPrefix is the prefix that consul keys will be assumed to have if you
+	// haven't specified one
+	DefaultPrefix = "/micro/config/"
+)
+
+func (c *consul) Read() (*source.ChangeSet, error) {
+	kv, _, err := c.client.KV().List(c.prefix, nil)
+	if err != nil {
+		return nil, err
+	}
+
+	if kv == nil || len(kv) == 0 {
+		return nil, fmt.Errorf("source not found: %s", c.prefix)
+	}
+
+	data, err := makeMap(c.opts.Encoder, kv, c.stripPrefix)
+	if err != nil {
+		return nil, fmt.Errorf("error reading data: %v", err)
+	}
+
+	b, err := c.opts.Encoder.Encode(data)
+	if err != nil {
+		return nil, fmt.Errorf("error reading source: %v", err)
+	}
+
+	cs := &source.ChangeSet{
+		Timestamp: time.Now(),
+		Format:    c.opts.Encoder.String(),
+		Source:    c.String(),
+		Data:      b,
+	}
+	cs.Checksum = cs.Sum()
+
+	return cs, nil
+}
+
+func (c *consul) String() string {
+	return "consul"
+}
+
+func (c *consul) Watch() (source.Watcher, error) {
+	w, err := newWatcher(c.prefix, c.addr, c.String(), c.stripPrefix, c.opts.Encoder)
+	if err != nil {
+		return nil, err
+	}
+	return w, nil
+}
+
+// NewSource creates a new consul source
+func NewSource(opts ...source.Option) source.Source {
+	options := source.NewOptions(opts...)
+
+	// use default config
+	config := api.DefaultConfig()
+
+	// check if there are any addrs
+	a, ok := options.Context.Value(addressKey{}).(string)
+	if ok {
+		addr, port, err := net.SplitHostPort(a)
+		if ae, ok := err.(*net.AddrError); ok && ae.Err == "missing port in address" {
+			port = "8500"
+			addr = a
+			config.Address = fmt.Sprintf("%s:%s", addr, port)
+		} else if err == nil {
+			config.Address = fmt.Sprintf("%s:%s", addr, port)
+		}
+	}
+
+	dc, ok := options.Context.Value(dcKey{}).(string)
+	if ok {
+		config.Datacenter = dc
+	}
+
+	token, ok := options.Context.Value(tokenKey{}).(string)
+	if ok {
+		config.Token = token
+	}
+
+	// create the client
+	client, _ := api.NewClient(config)
+
+	prefix := DefaultPrefix
+	sp := ""
+	f, ok := options.Context.Value(prefixKey{}).(string)
+	if ok {
+		prefix = f
+	}
+
+	if b, ok := options.Context.Value(stripPrefixKey{}).(bool); ok && b {
+		sp = prefix
+	}
+
+	return &consul{
+		prefix:      prefix,
+		stripPrefix: sp,
+		addr:        config.Address,
+		opts:        options,
+		client:      client,
+	}
+}

config/source/consul/options.go

@@ -0,0 +1,63 @@
+package consul
+
+import (
+	"context"
+
+	"github.com/micro/go-micro/config/source"
+)
+
+type addressKey struct{}
+type prefixKey struct{}
+type stripPrefixKey struct{}
+type dcKey struct{}
+type tokenKey struct{}
+
+// WithAddress sets the consul address
+func WithAddress(a string) source.Option {
+	return func(o *source.Options) {
+		if o.Context == nil {
+			o.Context = context.Background()
+		}
+		o.Context = context.WithValue(o.Context, addressKey{}, a)
+	}
+}
+
+// WithPrefix sets the key prefix to use
+func WithPrefix(p string) source.Option {
+	return func(o *source.Options) {
+		if o.Context == nil {
+			o.Context = context.Background()
+		}
+		o.Context = context.WithValue(o.Context, prefixKey{}, p)
+	}
+}
+
+// StripPrefix indicates whether to remove the prefix from config entries, or leave it in place.
+func StripPrefix(strip bool) source.Option {
+	return func(o *source.Options) {
+		if o.Context == nil {
+			o.Context = context.Background()
+		}
+
+		o.Context = context.WithValue(o.Context, stripPrefixKey{}, strip)
+	}
+}
+
+func WithDatacenter(p string) source.Option {
+	return func(o *source.Options) {
+		if o.Context == nil {
+			o.Context = context.Background()
+		}
+		o.Context = context.WithValue(o.Context, dcKey{}, p)
+	}
+}
+
+// WithToken sets the key token to use
+func WithToken(p string) source.Option {
+	return func(o *source.Options) {
+		if o.Context == nil {
+			o.Context = context.Background()
+		}
+		o.Context = context.WithValue(o.Context, tokenKey{}, p)
+	}
+}

config/source/consul/util.go

@@ -0,0 +1,49 @@
+package consul
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/hashicorp/consul/api"
+	"github.com/micro/go-micro/config/encoder"
+)
+
+func makeMap(e encoder.Encoder, kv api.KVPairs, stripPrefix string) (map[string]interface{}, error) {
+	data := make(map[string]interface{})
+
+	// consul guarantees lexicographic order, so no need to sort
+	for _, v := range kv {
+		pathString := strings.TrimPrefix(strings.TrimPrefix(v.Key, stripPrefix), "/")
+		var val map[string]interface{}
+
+		// ensure a valid value is stored at this location
+		if len(v.Value) > 0 {
+			if err := e.Decode(v.Value, &val); err != nil {
+				return nil, fmt.Errorf("faild decode value. path: %s, error: %s", pathString, err)
+			}
+		}
+
+		// set target at the root
+		target := data
+
+		// then descend to the target location, creating as we go, if need be
+		if pathString != "" {
+			path := strings.Split(pathString, "/")
+			// find (or create) the location we want to put this value at
+			for _, dir := range path {
+				if _, ok := target[dir]; !ok {
+					target[dir] = make(map[string]interface{})
+				}
+				target = target[dir].(map[string]interface{})
+			}
+
+		}
+
+		// copy over the keys from the value
+		for k := range val {
+			target[k] = val[k]
+		}
+	}
+
+	return data, nil
+}

config/source/consul/watcher.go

@@ -0,0 +1,96 @@
+package consul
+
+import (
+	"errors"
+	"time"
+
+	"github.com/hashicorp/consul/api"
+	"github.com/hashicorp/consul/api/watch"
+	"github.com/micro/go-micro/config/encoder"
+	"github.com/micro/go-micro/config/source"
+)
+
+type watcher struct {
+	e           encoder.Encoder
+	name        string
+	stripPrefix string
+
+	wp   *watch.Plan
+	ch   chan *source.ChangeSet
+	exit chan bool
+}
+
+func newWatcher(key, addr, name, stripPrefix string, e encoder.Encoder) (source.Watcher, error) {
+	w := &watcher{
+		e:           e,
+		name:        name,
+		stripPrefix: stripPrefix,
+		ch:          make(chan *source.ChangeSet),
+		exit:        make(chan bool),
+	}
+
+	wp, err := watch.Parse(map[string]interface{}{"type": "keyprefix", "prefix": key})
+	if err != nil {
+		return nil, err
+	}
+
+	wp.Handler = w.handle
+
+	// wp.Run is a blocking call and will prevent newWatcher from returning
+	go wp.Run(addr)
+
+	w.wp = wp
+
+	return w, nil
+}
+
+func (w *watcher) handle(idx uint64, data interface{}) {
+	if data == nil {
+		return
+	}
+
+	kvs, ok := data.(api.KVPairs)
+	if !ok {
+		return
+	}
+
+	d, err := makeMap(w.e, kvs, w.stripPrefix)
+	if err != nil {
+		return
+	}
+
+	b, err := w.e.Encode(d)
+	if err != nil {
+		return
+	}
+
+	cs := &source.ChangeSet{
+		Timestamp: time.Now(),
+		Format:    w.e.String(),
+		Source:    w.name,
+		Data:      b,
+	}
+	cs.Checksum = cs.Sum()
+
+	w.ch <- cs
+}
+
+func (w *watcher) Next() (*source.ChangeSet, error) {
+	select {
+	case cs := <-w.ch:
+		return cs, nil
+	case <-w.exit:
+		return nil, errors.New("watcher stopped")
+	}
+}
+
+func (w *watcher) Stop() error {
+	select {
+	case <-w.exit:
+		return nil
+	default:
+		w.wp.Stop()
+		close(w.exit)
+	}
+	return nil
+}

config/source/env/README.md

@@ -0,0 +1,96 @@
+# Env Source
+
+The env source reads config from environment variables
+
+## Format
+
+We expect environment variables to be in the standard format of FOO=bar
+
+Keys are converted to lowercase and split on underscore.
+
+
+### Example
+
+```
+DATABASE_ADDRESS=127.0.0.1
+DATABASE_PORT=3306
+```
+
+Becomes
+
+```json
+{
+    "database": {
+        "address": "127.0.0.1",
+        "port": 3306
+    }
+}
+```
+
+## Prefixes
+
+Environment variables can be namespaced so we only have access to a subset. Two options are available:
+
+```
+WithPrefix(p ...string)
+WithStrippedPrefix(p ...string)
+```
+
+The former will preserve the prefix and make it a top level key in the config. The latter eliminates the prefix, reducing the nesting by one. 
+
+#### Example:
+
+Given ENVs of:
+
+```
+APP_DATABASE_ADDRESS=127.0.0.1
+APP_DATABASE_PORT=3306
+VAULT_ADDR=vault:1337
+```
+
+and a source initialized as follows:
+
+```
+src := env.NewSource(
+    env.WithPrefix("VAULT"),
+    env.WithStrippedPrefix("APP"),
+)
+```
+
+The resulting config will be:
+
+```
+{
+    "database": {
+        "address": "127.0.0.1",
+        "port": 3306
+    },
+    "vault": {
+        "addr": "vault:1337"
+    }
+}
+```
+
+
+## New Source
+
+Specify source with data
+
+```go
+src := env.NewSource(
+	// optionally specify prefix
+	env.WithPrefix("MICRO"),
+)
+```
+
+## Load Source
+
+Load the source into config
+
+```go
+// Create new config
+conf := config.NewConfig()
+
+// Load file source
+conf.Load(src)
+```

config/source/env/env.go

@@ -0,0 +1,142 @@
+package env
+
+import (
+	"os"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/imdario/mergo"
+	"github.com/micro/go-micro/config/source"
+)
+
+var (
+	DefaultPrefixes = []string{}
+)
+
+type env struct {
+	prefixes         []string
+	strippedPrefixes []string
+	opts             source.Options
+}
+
+func (e *env) Read() (*source.ChangeSet, error) {
+	var changes map[string]interface{}
+
+	for _, env := range os.Environ() {
+
+		if len(e.prefixes) > 0 || len(e.strippedPrefixes) > 0 {
+			notFound := true
+
+			if _, ok := matchPrefix(e.prefixes, env); ok {
+				notFound = false
+			}
+
+			if match, ok := matchPrefix(e.strippedPrefixes, env); ok {
+				env = strings.TrimPrefix(env, match)
+				notFound = false
+			}
+
+			if notFound {
+				continue
+			}
+		}
+
+		pair := strings.SplitN(env, "=", 2)
+		value := pair[1]
+		keys := strings.Split(strings.ToLower(pair[0]), "_")
+		reverse(keys)
+
+		tmp := make(map[string]interface{})
+		for i, k := range keys {
+			if i == 0 {
+				if intValue, err := strconv.Atoi(value); err == nil {
+					tmp[k] = intValue
+				} else if boolValue, err := strconv.ParseBool(value); err == nil {
+					tmp[k] = boolValue
+				} else {
+					tmp[k] = value
+				}
+				continue
+			}
+
+			tmp = map[string]interface{}{k: tmp}
+		}
+
+		if err := mergo.Map(&changes, tmp); err != nil {
+			return nil, err
+		}
+	}
+
+	b, err := e.opts.Encoder.Encode(changes)
+	if err != nil {
+		return nil, err
+	}
+
+	cs := &source.ChangeSet{
+		Format:    e.opts.Encoder.String(),
+		Data:      b,
+		Timestamp: time.Now(),
+		Source:    e.String(),
+	}
+	cs.Checksum = cs.Sum()
+
+	return cs, nil
+}
+
+func matchPrefix(pre []string, s string) (string, bool) {
+	for _, p := range pre {
+		if strings.HasPrefix(s, p) {
+			return p, true
+		}
+	}
+
+	return "", false
+}
+
+func reverse(ss []string) {
+	for i := len(ss)/2 - 1; i >= 0; i-- {
+		opp := len(ss) - 1 - i
+		ss[i], ss[opp] = ss[opp], ss[i]
+	}
+}
+
+func (e *env) Watch() (source.Watcher, error) {
+	return newWatcher()
+}
+
+func (e *env) String() string {
+	return "env"
+}
+
+// NewSource returns a config source for parsing ENV variables.
+// Underscores are delimiters for nesting, and all keys are lowercased.
+//
+// Example:
+//      "DATABASE_SERVER_HOST=localhost" will convert to
+//
+//      {
+//          "database": {
+//              "server": {
+//                  "host": "localhost"
+//              }
+//          }
+//      }
+func NewSource(opts ...source.Option) source.Source {
+	options := source.NewOptions(opts...)
+
+	var sp []string
+	var pre []string
+	if p, ok := options.Context.Value(strippedPrefixKey{}).([]string); ok {
+		sp = p
+	}
+
+	if p, ok := options.Context.Value(prefixKey{}).([]string); ok {
+		pre = p
+	}
+
+	if len(sp) > 0 || len(pre) > 0 {
+		pre = append(pre, DefaultPrefixes...)
+	}
+	return &env{prefixes: pre, strippedPrefixes: sp, opts: options}
+}

config/source/env/env_test.go

@@ -0,0 +1,112 @@
+package env
+
+import (
+	"encoding/json"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/micro/go-micro/config/source"
+)
+
+func TestEnv_Read(t *testing.T) {
+	expected := map[string]map[string]string{
+		"database": {
+			"host":       "localhost",
+			"password":   "password",
+			"datasource": "user:password@tcp(localhost:port)/db?charset=utf8mb4&parseTime=True&loc=Local",
+		},
+	}
+
+	os.Setenv("DATABASE_HOST", "localhost")
+	os.Setenv("DATABASE_PASSWORD", "password")
+	os.Setenv("DATABASE_DATASOURCE", "user:password@tcp(localhost:port)/db?charset=utf8mb4&parseTime=True&loc=Local")
+
+	source := NewSource()
+	c, err := source.Read()
+	if err != nil {
+		t.Error(err)
+	}
+
+	var actual map[string]interface{}
+	if err := json.Unmarshal(c.Data, &actual); err != nil {
+		t.Error(err)
+	}
+
+	actualDB := actual["database"].(map[string]interface{})
+
+	for k, v := range expected["database"] {
+		a := actualDB[k]
+
+		if a != v {
+			t.Errorf("expected %v got %v", v, a)
+		}
+	}
+}
+
+func TestEnvvar_Prefixes(t *testing.T) {
+	os.Setenv("APP_DATABASE_HOST", "localhost")
+	os.Setenv("APP_DATABASE_PASSWORD", "password")
+	os.Setenv("VAULT_ADDR", "vault:1337")
+	os.Setenv("MICRO_REGISTRY", "mdns")
+
+	var prefixtests = []struct {
+		prefixOpts   []source.Option
+		expectedKeys []string
+	}{
+		{[]source.Option{WithPrefix("APP", "MICRO")}, []string{"app", "micro"}},
+		{[]source.Option{WithPrefix("MICRO"), WithStrippedPrefix("APP")}, []string{"database", "micro"}},
+		{[]source.Option{WithPrefix("MICRO"), WithStrippedPrefix("APP")}, []string{"database", "micro"}},
+	}
+
+	for _, pt := range prefixtests {
+		source := NewSource(pt.prefixOpts...)
+
+		c, err := source.Read()
+		if err != nil {
+			t.Error(err)
+		}
+
+		var actual map[string]interface{}
+		if err := json.Unmarshal(c.Data, &actual); err != nil {
+			t.Error(err)
+		}
+
+		// assert other prefixes ignored
+		if l := len(actual); l != len(pt.expectedKeys) {
+			t.Errorf("expected %v top keys, got %v", len(pt.expectedKeys), l)
+		}
+
+		for _, k := range pt.expectedKeys {
+			if !containsKey(actual, k) {
+				t.Errorf("expected key %v, not found", k)
+			}
+		}
+	}
+}
+
+func TestEnvvar_WatchNextNoOpsUntilStop(t *testing.T) {
+	source := NewSource(WithStrippedPrefix("GOMICRO_"))
+	w, err := source.Watch()
+	if err != nil {
+		t.Error(err)
+	}
+
+	go func() {
+		time.Sleep(50 * time.Millisecond)
+		w.Stop()
+	}()
+
+	if _, err := w.Next(); err.Error() != "watcher stopped" {
+		t.Errorf("expected watcher stopped error, got %v", err)
+	}
+}
+
+func containsKey(m map[string]interface{}, s string) bool {
+	for k := range m {
+		if k == s {
+			return true
+		}
+	}
+	return false
+}

config/source/env/options.go

@@ -0,0 +1,49 @@
+package env
+
+import (
+	"context"
+
+	"strings"
+
+	"github.com/micro/go-micro/config/source"
+)
+
+type strippedPrefixKey struct{}
+type prefixKey struct{}
+
+// WithStrippedPrefix sets the environment variable prefixes to scope to.
+// These prefixes will be removed from the actual config entries.
+func WithStrippedPrefix(p ...string) source.Option {
+	return func(o *source.Options) {
+		if o.Context == nil {
+			o.Context = context.Background()
+		}
+
+		o.Context = context.WithValue(o.Context, strippedPrefixKey{}, appendUnderscore(p))
+	}
+}
+
+// WithPrefix sets the environment variable prefixes to scope to.
+// These prefixes will not be removed. Each prefix will be considered a top level config entry.
+func WithPrefix(p ...string) source.Option {
+	return func(o *source.Options) {
+		if o.Context == nil {
+			o.Context = context.Background()
+		}
+		o.Context = context.WithValue(o.Context, prefixKey{}, appendUnderscore(p))
+	}
+}
+
+func appendUnderscore(prefixes []string) []string {
+	var result []string
+	for _, p := range prefixes {
+		if !strings.HasSuffix(p, "_") {
+			result = append(result, p+"_")
+			continue
+		}
+
+		result = append(result, p)
+	}
+
+	return result
+}

config/source/env/watcher.go

@@ -0,0 +1,26 @@
+package env
+
+import (
+	"errors"
+
+	"github.com/micro/go-micro/config/source"
+)
+
+type watcher struct {
+	exit chan struct{}
+}
+
+func (w *watcher) Next() (*source.ChangeSet, error) {
+	<-w.exit
+
+	return nil, errors.New("watcher stopped")
+}
+
+func (w *watcher) Stop() error {
+	close(w.exit)
+	return nil
+}
+
+func newWatcher() (source.Watcher, error) {
+	return &watcher{exit: make(chan struct{})}, nil
+}

config/source/file/README.md

@@ -0,0 +1,70 @@
+# File Source
+
+The file source reads config from a file. 
+
+It uses the File extension to determine the Format e.g `config.yaml` has the yaml format. 
+It does not make use of encoders or interpet the file data. If a file extension is not present 
+the source Format will default to the Encoder in options.
+
+## Example
+
+A config file format in json
+
+```json
+{
+    "hosts": {
+        "database": {
+            "address": "10.0.0.1",
+            "port": 3306
+        },
+        "cache": {
+            "address": "10.0.0.2",
+            "port": 6379
+        }
+    }
+}
+```
+
+## New Source
+
+Specify file source with path to file. Path is optional and will default to `config.json`
+
+```go
+fileSource := file.NewSource(
+	file.WithPath("/tmp/config.json"),
+)
+```
+
+## File Format
+
+To load different file formats e.g yaml, toml, xml simply specify them with their extension
+
+```
+fileSource := file.NewSource(
+        file.WithPath("/tmp/config.yaml"),
+)
+```
+
+If you want to specify a file without extension, ensure you set the encoder to the same format
+
+```
+e := toml.NewEncoder()
+
+fileSource := file.NewSource(
+        file.WithPath("/tmp/config"),
+	source.WithEncoder(e),
+)
+```
+
+## Load Source
+
+Load the source into config
+
+```go
+// Create new config
+conf := config.NewConfig()
+
+// Load file source
+conf.Load(fileSource)
+```
+

config/source/file/file.go

@@ -0,0 +1,65 @@
+// Package file is a file source. Expected format is json
+package file
+
+import (
+	"io/ioutil"
+	"os"
+
+	"github.com/micro/go-micro/config/source"
+)
+
+type file struct {
+	path string
+	opts source.Options
+}
+
+var (
+	DefaultPath = "config.json"
+)
+
+func (f *file) Read() (*source.ChangeSet, error) {
+	fh, err := os.Open(f.path)
+	if err != nil {
+		return nil, err
+	}
+	defer fh.Close()
+	b, err := ioutil.ReadAll(fh)
+	if err != nil {
+		return nil, err
+	}
+	info, err := fh.Stat()
+	if err != nil {
+		return nil, err
+	}
+
+	cs := &source.ChangeSet{
+		Format:    format(f.path, f.opts.Encoder),
+		Source:    f.String(),
+		Timestamp: info.ModTime(),
+		Data:      b,
+	}
+	cs.Checksum = cs.Sum()
+
+	return cs, nil
+}
+
+func (f *file) String() string {
+	return "file"
+}
+
+func (f *file) Watch() (source.Watcher, error) {
+	if _, err := os.Stat(f.path); err != nil {
+		return nil, err
+	}
+	return newWatcher(f)
+}
+
+func NewSource(opts ...source.Option) source.Source {
+	options := source.NewOptions(opts...)
+	path := DefaultPath
+	f, ok := options.Context.Value(filePathKey{}).(string)
+	if ok {
+		path = f
+	}
+	return &file{opts: options, path: path}
+}

config/source/file/file_test.go

@@ -0,0 +1,37 @@
+package file
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"testing"
+	"time"
+)
+
+func TestFile(t *testing.T) {
+	data := []byte(`{"foo": "bar"}`)
+	path := filepath.Join(os.TempDir(), fmt.Sprintf("file.%d", time.Now().UnixNano()))
+	fh, err := os.Create(path)
+	if err != nil {
+		t.Error(err)
+	}
+	defer func() {
+		fh.Close()
+		os.Remove(path)
+	}()
+
+	_, err = fh.Write(data)
+	if err != nil {
+		t.Error(err)
+	}
+
+	f := NewSource(WithPath(path))
+	c, err := f.Read()
+	if err != nil {
+		t.Error(err)
+	}
+	t.Logf("%+v", c)
+	if string(c.Data) != string(data) {
+		t.Error("data from file does not match")
+	}
+}

config/source/file/format.go

@@ -0,0 +1,15 @@
+package file
+
+import (
+	"strings"
+
+	"github.com/micro/go-micro/config/encoder"
+)
+
+func format(p string, e encoder.Encoder) string {
+	parts := strings.Split(p, ".")
+	if len(parts) > 1 {
+		return parts[len(parts)-1]
+	}
+	return e.String()
+}

config/source/file/format_test.go

@@ -0,0 +1,31 @@
+package file
+
+import (
+	"testing"
+
+	"github.com/micro/go-micro/config/source"
+)
+
+func TestFormat(t *testing.T) {
+	opts := source.NewOptions()
+	e := opts.Encoder
+
+	testCases := []struct {
+		p string
+		f string
+	}{
+		{"/foo/bar.json", "json"},
+		{"/foo/bar.yaml", "yaml"},
+		{"/foo/bar.xml", "xml"},
+		{"/foo/bar.conf.ini", "ini"},
+		{"conf", e.String()},
+	}
+
+	for _, d := range testCases {
+		f := format(d.p, e)
+		if f != d.f {
+			t.Fatalf("%s: expected %s got %s", d.p, d.f, f)
+		}
+	}
+
+}

config/source/file/options.go

@@ -0,0 +1,19 @@
+package file
+
+import (
+	"context"
+
+	"github.com/micro/go-micro/config/source"
+)
+
+type filePathKey struct{}
+
+// WithPath sets the path to file
+func WithPath(p string) source.Option {
+	return func(o *source.Options) {
+		if o.Context == nil {
+			o.Context = context.Background()
+		}
+		o.Context = context.WithValue(o.Context, filePathKey{}, p)
+	}
+}

config/source/file/watcher.go

@@ -0,0 +1,66 @@
+package file
+
+import (
+	"errors"
+	"os"
+
+	"github.com/fsnotify/fsnotify"
+	"github.com/micro/go-micro/config/source"
+)
+
+type watcher struct {
+	f *file
+
+	fw   *fsnotify.Watcher
+	exit chan bool
+}
+
+func newWatcher(f *file) (source.Watcher, error) {
+	fw, err := fsnotify.NewWatcher()
+	if err != nil {
+		return nil, err
+	}
+
+	fw.Add(f.path)
+
+	return &watcher{
+		f:    f,
+		fw:   fw,
+		exit: make(chan bool),
+	}, nil
+}
+
+func (w *watcher) Next() (*source.ChangeSet, error) {
+	// is it closed?
+	select {
+	case <-w.exit:
+		return nil, errors.New("watcher stopped")
+	default:
+	}
+
+	// try get the event
+	select {
+	case event, _ := <-w.fw.Events:
+		if event.Op == fsnotify.Rename {
+			// check existence of file, and add watch again
+			_, err := os.Stat(event.Name)
+			if err == nil || os.IsExist(err) {
+				w.fw.Add(event.Name)
+			}
+		}
+
+		c, err := w.f.Read()
+		if err != nil {
+			return nil, err
+		}
+		return c, nil
+	case err := <-w.fw.Errors:
+		return nil, err
+	case <-w.exit:
+		return nil, errors.New("watcher stopped")
+	}
+}
+
+func (w *watcher) Stop() error {
+	return w.fw.Close()
+}

config/source/flag/README.md

@@ -0,0 +1,47 @@
+# Flag Source
+
+The flag source reads config from flags
+
+## Format
+
+We expect the use of the `flag` package. Upper case flags will be lower cased. Dashes will be used as delimiters.
+
+### Example
+
+```
+dbAddress := flag.String("database_address", "127.0.0.1", "the db address")
+dbPort := flag.Int("database_port", 3306, "the db port)
+```
+
+Becomes
+
+```json
+{
+    "database": {
+        "address": "127.0.0.1",
+        "port": 3306
+    }
+}
+```
+
+## New Source
+
+```go
+flagSource := flag.NewSource(
+	// optionally enable reading of unset flags and their default
+	// values into config, defaults to false
+	IncludeUnset(true)
+)
+```
+
+## Load Source
+
+Load the source into config
+
+```go
+// Create new config
+conf := config.NewConfig()
+
+// Load file source
+conf.Load(flagSource)
+```

config/source/flag/flag.go

@@ -0,0 +1,97 @@
+package flag
+
+import (
+	"errors"
+	"flag"
+	"github.com/imdario/mergo"
+	"github.com/micro/go-micro/config/source"
+	"strings"
+	"time"
+)
+
+type flagsrc struct {
+	opts source.Options
+}
+
+func (fs *flagsrc) Read() (*source.ChangeSet, error) {
+	if !flag.Parsed() {
+		return nil, errors.New("flags not parsed")
+	}
+
+	var changes map[string]interface{}
+
+	visitFn := func(f *flag.Flag) {
+		n := strings.ToLower(f.Name)
+		keys := strings.FieldsFunc(n, split)
+		reverse(keys)
+
+		tmp := make(map[string]interface{})
+		for i, k := range keys {
+			if i == 0 {
+				tmp[k] = f.Value
+				continue
+			}
+
+			tmp = map[string]interface{}{k: tmp}
+		}
+
+		mergo.Map(&changes, tmp) // need to sort error handling
+		return
+	}
+
+	unset, ok := fs.opts.Context.Value(includeUnsetKey{}).(bool)
+	if ok && unset {
+		flag.VisitAll(visitFn)
+	} else {
+		flag.Visit(visitFn)
+	}
+
+	b, err := fs.opts.Encoder.Encode(changes)
+	if err != nil {
+		return nil, err
+	}
+
+	cs := &source.ChangeSet{
+		Format:    fs.opts.Encoder.String(),
+		Data:      b,
+		Timestamp: time.Now(),
+		Source:    fs.String(),
+	}
+	cs.Checksum = cs.Sum()
+
+	return cs, nil
+}
+
+func split(r rune) bool {
+	return r == '-' || r == '_'
+}
+
+func reverse(ss []string) {
+	for i := len(ss)/2 - 1; i >= 0; i-- {
+		opp := len(ss) - 1 - i
+		ss[i], ss[opp] = ss[opp], ss[i]
+	}
+}
+
+func (fs *flagsrc) Watch() (source.Watcher, error) {
+	return source.NewNoopWatcher()
+}
+
+func (fs *flagsrc) String() string {
+	return "flag"
+}
+
+// NewSource returns a config source for integrating parsed flags.
+// Hyphens are delimiters for nesting, and all keys are lowercased.
+//
+// Example:
+//      dbhost := flag.String("database-host", "localhost", "the db host name")
+//
+//      {
+//          "database": {
+//              "host": "localhost"
+//          }
+//      }
+func NewSource(opts ...source.Option) source.Source {
+	return &flagsrc{opts: source.NewOptions(opts...)}
+}

config/source/flag/flag_test.go

@@ -0,0 +1,66 @@
+package flag
+
+import (
+	"encoding/json"
+	"flag"
+	"testing"
+)
+
+var (
+	dbuser = flag.String("database-user", "default", "db user")
+	dbhost = flag.String("database-host", "", "db host")
+	dbpw   = flag.String("database-password", "", "db pw")
+)
+
+func init() {
+	flag.Set("database-host", "localhost")
+	flag.Set("database-password", "some-password")
+	flag.Parse()
+}
+
+func TestFlagsrc_Read(t *testing.T) {
+	source := NewSource()
+	c, err := source.Read()
+	if err != nil {
+		t.Error(err)
+	}
+
+	var actual map[string]interface{}
+	if err := json.Unmarshal(c.Data, &actual); err != nil {
+		t.Error(err)
+	}
+
+	actualDB := actual["database"].(map[string]interface{})
+	if actualDB["host"] != *dbhost {
+		t.Errorf("expected %v got %v", *dbhost, actualDB["host"])
+	}
+
+	if actualDB["password"] != *dbpw {
+		t.Errorf("expected %v got %v", *dbpw, actualDB["password"])
+	}
+
+	// unset flags should not be loaded
+	if actualDB["user"] != nil {
+		t.Errorf("expected %v got %v", nil, actualDB["user"])
+	}
+}
+
+func TestFlagsrc_ReadAll(t *testing.T) {
+	source := NewSource(IncludeUnset(true))
+	c, err := source.Read()
+	if err != nil {
+		t.Error(err)
+	}
+
+	var actual map[string]interface{}
+	if err := json.Unmarshal(c.Data, &actual); err != nil {
+		t.Error(err)
+	}
+
+	actualDB := actual["database"].(map[string]interface{})
+
+	// unset flag defaults should be loaded
+	if actualDB["user"] != *dbuser {
+		t.Errorf("expected %v got %v", *dbuser, actualDB["user"])
+	}
+}

config/source/flag/options.go

@@ -0,0 +1,20 @@
+package flag
+
+import (
+	"context"
+
+	"github.com/micro/go-micro/config/source"
+)
+
+type includeUnsetKey struct{}
+
+// IncludeUnset toggles the loading of unset flags and their respective default values.
+// Default behavior is to ignore any unset flags.
+func IncludeUnset(b bool) source.Option {
+	return func(o *source.Options) {
+		if o.Context == nil {
+			o.Context = context.Background()
+		}
+		o.Context = context.WithValue(o.Context, includeUnsetKey{}, true)
+	}
+}

config/source/memory/README.md

@@ -0,0 +1,44 @@
+# Memory Source
+
+The memory source provides in-memory data as a source
+
+## Memory Format
+
+The expected data format is json
+
+```json
+data := []byte(`{
+    "hosts": {
+        "database": {
+            "address": "10.0.0.1",
+            "port": 3306
+        },
+        "cache": {
+            "address": "10.0.0.2",
+            "port": 6379
+        }
+    }
+}`)
+```
+
+## New Source
+
+Specify source with data
+
+```go
+memorySource := memory.NewSource(
+	memory.WithData(data),
+)
+```
+
+## Load Source
+
+Load the source into config
+
+```go
+// Create new config
+conf := config.NewConfig()
+
+// Load file source
+conf.Load(memorySource)
+```

config/source/memory/memory.go

@@ -0,0 +1,93 @@
+// Package memory is a memory source
+package memory
+
+import (
+	"sync"
+	"time"
+
+	"github.com/micro/go-micro/config/source"
+	"github.com/pborman/uuid"
+)
+
+type memory struct {
+	sync.RWMutex
+	ChangeSet *source.ChangeSet
+	Watchers  map[string]*watcher
+}
+
+func (s *memory) Read() (*source.ChangeSet, error) {
+	s.RLock()
+	cs := &source.ChangeSet{
+		Timestamp: s.ChangeSet.Timestamp,
+		Data:      s.ChangeSet.Data,
+		Checksum:  s.ChangeSet.Checksum,
+		Source:    s.ChangeSet.Source,
+	}
+	s.RUnlock()
+	return cs, nil
+}
+
+func (s *memory) Watch() (source.Watcher, error) {
+	w := &watcher{
+		Id:      uuid.NewUUID().String(),
+		Updates: make(chan *source.ChangeSet, 100),
+		Source:  s,
+	}
+
+	s.Lock()
+	s.Watchers[w.Id] = w
+	s.Unlock()
+	return w, nil
+}
+
+// Update allows manual updates of the config data.
+func (s *memory) Update(c *source.ChangeSet) {
+	// don't process nil
+	if c == nil {
+		return
+	}
+
+	// hash the file
+	s.Lock()
+	// update changeset
+	s.ChangeSet = &source.ChangeSet{
+		Data:      c.Data,
+		Format:    c.Format,
+		Source:    "memory",
+		Timestamp: time.Now(),
+	}
+	s.ChangeSet.Checksum = s.ChangeSet.Sum()
+
+	// update watchers
+	for _, w := range s.Watchers {
+		select {
+		case w.Updates <- s.ChangeSet:
+		default:
+		}
+	}
+	s.Unlock()
+}
+
+func (s *memory) String() string {
+	return "memory"
+}
+
+func NewSource(opts ...source.Option) source.Source {
+	var options source.Options
+	for _, o := range opts {
+		o(&options)
+	}
+
+	s := &memory{
+		Watchers: make(map[string]*watcher),
+	}
+
+	if options.Context != nil {
+		c, ok := options.Context.Value(changeSetKey{}).(*source.ChangeSet)
+		if ok {
+			s.Update(c)
+		}
+	}
+
+	return s
+}

config/source/memory/options.go

@@ -0,0 +1,32 @@
+package memory
+
+import (
+	"context"
+
+	"github.com/micro/go-micro/config/source"
+)
+
+type changeSetKey struct{}
+
+// WithChangeSet allows a changeset to be set
+func WithChangeSet(cs *source.ChangeSet) source.Option {
+	return func(o *source.Options) {
+		if o.Context == nil {
+			o.Context = context.Background()
+		}
+		o.Context = context.WithValue(o.Context, changeSetKey{}, cs)
+	}
+}
+
+// WithData allows the source data to be set
+func WithData(d []byte) source.Option {
+	return func(o *source.Options) {
+		if o.Context == nil {
+			o.Context = context.Background()
+		}
+		o.Context = context.WithValue(o.Context, changeSetKey{}, &source.ChangeSet{
+			Data:   d,
+			Format: "json",
+		})
+	}
+}

config/source/memory/watcher.go

@@ -0,0 +1,23 @@
+package memory
+
+import (
+	"github.com/micro/go-micro/config/source"
+)
+
+type watcher struct {
+	Id      string
+	Updates chan *source.ChangeSet
+	Source  *memory
+}
+
+func (w *watcher) Next() (*source.ChangeSet, error) {
+	cs := <-w.Updates
+	return cs, nil
+}
+
+func (w *watcher) Stop() error {
+	w.Source.Lock()
+	delete(w.Source.Watchers, w.Id)
+	w.Source.Unlock()
+	return nil
+}

config/source/noop.go

@@ -0,0 +1,25 @@
+package source
+
+import (
+	"errors"
+)
+
+type noopWatcher struct {
+	exit chan struct{}
+}
+
+func (w *noopWatcher) Next() (*ChangeSet, error) {
+	<-w.exit
+
+	return nil, errors.New("noopWatcher stopped")
+}
+
+func (w *noopWatcher) Stop() error {
+	close(w.exit)
+	return nil
+}
+
+// NewNoopWatcher returns a watcher that blocks on Next() until Stop() is called.
+func NewNoopWatcher() (Watcher, error) {
+	return &noopWatcher{exit: make(chan struct{})}, nil
+}

config/source/options.go

@@ -0,0 +1,38 @@
+package source
+
+import (
+	"context"
+
+	"github.com/micro/go-micro/config/encoder"
+	"github.com/micro/go-micro/config/encoder/json"
+)
+
+type Options struct {
+	// Encoder
+	Encoder encoder.Encoder
+
+	// for alternative data
+	Context context.Context
+}
+
+type Option func(o *Options)
+
+func NewOptions(opts ...Option) Options {
+	options := Options{
+		Encoder: json.NewEncoder(),
+		Context: context.Background(),
+	}
+
+	for _, o := range opts {
+		o(&options)
+	}
+
+	return options
+}
+
+// WithEncoder sets the source encoder
+func WithEncoder(e encoder.Encoder) Option {
+	return func(o *Options) {
+		o.Encoder = e
+	}
+}

config/source/source.go

@@ -0,0 +1,28 @@
+// Package source is the interface for sources
+package source
+
+import (
+	"time"
+)
+
+// Source is the source from which config is loaded
+type Source interface {
+	Read() (*ChangeSet, error)
+	Watch() (Watcher, error)
+	String() string
+}
+
+// ChangeSet represents a set of changes from a source
+type ChangeSet struct {
+	Data      []byte
+	Checksum  string
+	Format    string
+	Source    string
+	Timestamp time.Time
+}
+
+// Watcher watches a source for changes
+type Watcher interface {
+	Next() (*ChangeSet, error)
+	Stop() error
+}