Secret implementation of config. Supporting config merge (#2027)

Co-authored-by: Asim Aslam <asim@aslam.me>

config/config.go

@@ -27,16 +27,17 @@ type Value interface {
 }
 
 type Options struct {
-	// Is the value being read a secret?
-	// If true, the Config will try to decode it with `SecretKey`
 	Secret bool
 }
 
-// Option sets values in Options
 type Option func(o *Options)
 
-func Secret(isSecret bool) Option {
+func Secret(b bool) Option {
 	return func(o *Options) {
-		o.Secret = isSecret
+		o.Secret = b
 	}
 }
+
+type Secrets interface {
+	Config
+}

config/secrets/encryption.go

@@ -0,0 +1,70 @@
+package secrets
+
+import (
+	"crypto/aes"
+	"crypto/cipher"
+	"crypto/rand"
+	"encoding/hex"
+	"fmt"
+	"io"
+)
+
+// encrypt/decrypt functions are taken from https://www.melvinvivas.com/how-to-encrypt-and-decrypt-data-using-aes/
+
+func encrypt(stringToEncrypt string, key []byte) (string, error) {
+	plaintext := []byte(stringToEncrypt)
+
+	//Create a new Cipher Block from the key
+	block, err := aes.NewCipher(key)
+	if err != nil {
+		return "", err
+	}
+
+	//Create a new GCM - https://en.wikipedia.org/wiki/Galois/Counter_Mode
+	//https://golang.org/pkg/crypto/cipher/#NewGCM
+	aesGCM, err := cipher.NewGCM(block)
+	if err != nil {
+		return "", err
+	}
+
+	//Create a nonce. Nonce should be from GCM
+	nonce := make([]byte, aesGCM.NonceSize())
+	if _, err = io.ReadFull(rand.Reader, nonce); err != nil {
+		return "", err
+	}
+
+	//Encrypt the data using aesGCM.Seal
+	//Since we don't want to save the nonce somewhere else in this case, we add it as a prefix to the encrypted data. The first nonce argument in Seal is the prefix.
+	ciphertext := aesGCM.Seal(nonce, nonce, plaintext, nil)
+	return fmt.Sprintf("%x", ciphertext), nil
+}
+
+func decrypt(encryptedString string, key []byte) (string, error) {
+	enc, _ := hex.DecodeString(encryptedString)
+
+	//Create a new Cipher Block from the key
+	block, err := aes.NewCipher(key)
+	if err != nil {
+		return "", err
+	}
+
+	//Create a new GCM
+	aesGCM, err := cipher.NewGCM(block)
+	if err != nil {
+		return "", err
+	}
+
+	//Get the nonce size
+	nonceSize := aesGCM.NonceSize()
+
+	//Extract the nonce from the encrypted data
+	nonce, ciphertext := enc[:nonceSize], enc[nonceSize:]
+
+	//Decrypt the data
+	plaintext, err := aesGCM.Open(nil, nonce, ciphertext, nil)
+	if err != nil {
+		return "", err
+	}
+
+	return fmt.Sprintf("%s", plaintext), nil
+}

config/secrets/secrets.go

@@ -0,0 +1,121 @@
+package secrets
+
+import (
+	"encoding/base64"
+	"encoding/json"
+	"fmt"
+
+	"github.com/micro/go-micro/v3/config"
+)
+
+// NewSecrets returns a config that encrypts values at rest
+func NewSecrets(config config.Config, encryptionKey string) (config.Secrets, error) {
+	return newSecrets(config, encryptionKey)
+}
+
+type secretConf struct {
+	config        config.Config
+	encryptionKey string
+}
+
+func newSecrets(config config.Config, encryptionKey string) (*secretConf, error) {
+	return &secretConf{
+		config:        config,
+		encryptionKey: encryptionKey,
+	}, nil
+}
+
+func (c *secretConf) Get(path string, options ...config.Option) (config.Value, error) {
+	val, err := c.config.Get(path, options...)
+	empty := config.NewJSONValue([]byte("null"))
+	if err != nil {
+		return empty, err
+	}
+	var v interface{}
+	err = json.Unmarshal(val.Bytes(), &v)
+	if err != nil {
+		return empty, err
+	}
+	v, err = convertElements(v, c.fromEncrypted)
+	if err != nil {
+		return empty, err
+	}
+	dat, err := json.Marshal(v)
+	if err != nil {
+		return empty, err
+	}
+	return config.NewJSONValue(dat), nil
+}
+
+func (c *secretConf) Set(path string, val interface{}, options ...config.Option) error {
+	// marshal to JSON and back so we can iterate on the
+	// value without reflection
+	JSON, err := json.Marshal(val)
+	if err != nil {
+		return err
+	}
+	var v interface{}
+	err = json.Unmarshal(JSON, &v)
+	if err != nil {
+		return err
+	}
+	v, err = convertElements(v, c.toEncrypted)
+	if err != nil {
+		return err
+	}
+	return c.config.Set(path, v)
+}
+
+func (c *secretConf) Delete(path string, options ...config.Option) error {
+	return c.config.Delete(path, options...)
+}
+
+func convertElements(elem interface{}, conversionFunc func(elem interface{}) (interface{}, error)) (interface{}, error) {
+	switch m := elem.(type) {
+	case map[string]interface{}:
+		for k, v := range m {
+			conv, err := convertElements(v, conversionFunc)
+			if err != nil {
+				return nil, err
+			}
+			m[k] = conv
+
+		}
+		return m, nil
+	}
+
+	return conversionFunc(elem)
+}
+
+func (c *secretConf) toEncrypted(elem interface{}) (interface{}, error) {
+	dat, err := json.Marshal(elem)
+	if err != nil {
+		return nil, err
+	}
+	encrypted, err := encrypt(string(dat), []byte(c.encryptionKey))
+	if err != nil {
+		return nil, fmt.Errorf("Failed to encrypt: %v", err)
+	}
+	return string(base64.StdEncoding.EncodeToString([]byte(encrypted))), nil
+}
+
+func (c *secretConf) fromEncrypted(elem interface{}) (interface{}, error) {
+	s, ok := elem.(string)
+	if !ok {
+		// This bit decides if the Secrets implementation suppports nonencrypted values
+		// ie. we could do:
+		// return nil, fmt.Errorf("Encrypted values should be strings, but got: %v", elem)
+		// but let's go with not making nonencrypted values blow up the whole thing
+		return elem, nil
+	}
+	dec, err := base64.StdEncoding.DecodeString(s)
+	if err != nil {
+		return elem, nil
+	}
+	decrypted, err := decrypt(string(dec), []byte(c.encryptionKey))
+	if err != nil {
+		return elem, nil
+	}
+	var ret interface{}
+	return ret, json.Unmarshal([]byte(decrypted), &ret)
+}

config/store/store.go

@@ -1,6 +1,9 @@
-package storeconfig
+package store
 
 import (
+	"encoding/json"
+	"strings"
+
 	"github.com/micro/go-micro/v3/config"
 	"github.com/micro/go-micro/v3/store"
 )
@@ -22,16 +25,6 @@ func newConfig(store store.Store, key string) (*conf, error) {
 	}, nil
 }
 
-func mergeOptions(old config.Options, nu ...config.Option) config.Options {
-	n := config.Options{
-		Secret: old.Secret,
-	}
-	for _, opt := range nu {
-		opt(&n)
-	}
-	return n
-}
-
 func (c *conf) Get(path string, options ...config.Option) (config.Value, error) {
 	rec, err := c.store.Read(c.key)
 	dat := []byte("{}")
@@ -49,13 +42,56 @@ func (c *conf) Set(path string, val interface{}, options ...config.Option) error
 		dat = rec[0].Value
 	}
 	values := config.NewJSONValues(dat)
-	values.Set(path, val)
+
+	// marshal to JSON and back so we can iterate on the
+	// value without reflection
+	// @todo only do this if a struct
+	JSON, err := json.Marshal(val)
+	if err != nil {
+		return err
+	}
+	var v interface{}
+	err = json.Unmarshal(JSON, &v)
+	if err != nil {
+		return err
+	}
+
+	m, ok := v.(map[string]interface{})
+	if ok {
+		err := traverse(m, []string{path}, func(p string, value interface{}) error {
+			values.Set(p, value)
+			return nil
+		})
+		if err != nil {
+			return err
+		}
+	} else {
+		values.Set(path, val)
+	}
 	return c.store.Write(&store.Record{
 		Key:   c.key,
 		Value: values.Bytes(),
 	})
 }
 
+func traverse(m map[string]interface{}, paths []string, callback func(path string, value interface{}) error) error {
+	for k, v := range m {
+		val, ok := v.(map[string]interface{})
+		if !ok {
+			err := callback(strings.Join(append(paths, k), "."), v)
+			if err != nil {
+				return err
+			}
+			continue
+		}
+		err := traverse(val, append(paths, k), callback)
+		if err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
 func (c *conf) Delete(path string, options ...config.Option) error {
 	rec, err := c.store.Read(c.key)
 	dat := []byte("{}")

config/store/store_test.go

@@ -0,0 +1,119 @@
+package store
+
+import (
+	"reflect"
+	"testing"
+
+	"github.com/micro/go-micro/v3/config"
+	"github.com/micro/go-micro/v3/config/secrets"
+	"github.com/micro/go-micro/v3/store/memory"
+)
+
+type conf1 struct {
+	A string  `json:"a"`
+	B int64   `json:"b"`
+	C float64 `json:"c"`
+	D bool    `json:"d"`
+}
+
+func TestBasics(t *testing.T) {
+	conf, err := NewConfig(memory.NewStore(), "micro")
+	if err != nil {
+		t.Fatal(err)
+	}
+	testBasics(conf, t)
+	// We need to get a new config because existing config so
+	conf, err = NewConfig(memory.NewStore(), "micro1")
+	if err != nil {
+		t.Fatal(err)
+	}
+	secrets, err := secrets.NewSecrets(conf, "somethingRandomButLongEnough32by")
+	if err != nil {
+		t.Fatal(err)
+	}
+	testBasics(secrets, t)
+}
+
+func testBasics(c config.Config, t *testing.T) {
+	original := &conf1{
+		"Hi", int64(42), float64(42.2), true,
+	}
+	err := c.Set("key", original)
+	if err != nil {
+		t.Fatal(err)
+	}
+	getted := &conf1{}
+	val, err := c.Get("key")
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = val.Scan(getted)
+	if err != nil {
+		t.Fatal(err)
+	}
+	if !reflect.DeepEqual(original, getted) {
+		t.Fatalf("Not equal: %v and %v", original, getted)
+	}
+
+	// Testing merges now
+	err = c.Set("key", map[string]interface{}{
+		"b": 55,
+		"e": map[string]interface{}{
+			"e1": true,
+		},
+	})
+	if err != nil {
+		t.Fatal(err)
+	}
+	m := map[string]interface{}{}
+	val, err = c.Get("key")
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = val.Scan(&m)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	expected := map[string]interface{}{
+		"a": "Hi",
+		"b": float64(55),
+		"c": float64(42.2),
+		"d": true,
+		"e": map[string]interface{}{
+			"e1": true,
+		},
+	}
+	if !reflect.DeepEqual(m, expected) {
+		t.Fatalf("Not equal: %v and %v", m, expected)
+	}
+
+	// Set just one value
+	expected = map[string]interface{}{
+		"a": "Hi",
+		"b": float64(55),
+		"c": float64(42.2),
+		"d": true,
+		"e": map[string]interface{}{
+			"e1": float64(45),
+		},
+	}
+	err = c.Set("key.e.e1", 45)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	m = map[string]interface{}{}
+	val, err = c.Get("key")
+	if err != nil {
+		t.Fatal(err)
+	}
+	err = val.Scan(&m)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if !reflect.DeepEqual(m, expected) {
+		t.Fatalf("Not equal: %v and %v", m, expected)
+	}
+}