unistack-org/micro
4
0
Fork 2
Code Issues 18 Pull Requests Actions Packages Projects Releases 324 Wiki Activity

Config interface change (#2010)

Browse Source
This commit is contained in:
Janos Dobronszki
2020-09-21 17:45:45 +02:00
committed by GitHub
parent 8975184b88
commit 9dbd75f2cc
50 changed files with 486 additions and 3311 deletions
Download Patch File Download Diff File Expand all files Collapse all files

56
config/config.go
View File

@@ -2,49 +2,41 @@
package config
import (
"context"
"github.com/micro/go-micro/v3/config/loader"
"github.com/micro/go-micro/v3/config/reader"
"github.com/micro/go-micro/v3/config/source"
"time"
)
// Config is an interface abstraction for dynamic configuration
type Config interface {
// provide the reader.Values interface
reader.Values
// Init the config
Init(opts ...Option) error
// Options in the config
Options() Options
// Stop the config loader/watcher
Close() error
// Load config sources
Load(source ...source.Source) error
// Force a source changeset sync
Sync() error
// Watch a value for changes
Watch(path ...string) (Watcher, error)
Get(path string, options ...Option) Value
Set(path string, val interface{}, options ...Option)
Delete(path string, options ...Option)
}
// Watcher is the config watcher
type Watcher interface {
Next() (reader.Value, error)
Stop() error
// Value represents a value of any type
type Value interface {
Exists() bool
Bool(def bool) bool
Int(def int) int
String(def string) string
Float64(def float64) float64
Duration(def time.Duration) time.Duration
StringSlice(def []string) []string
StringMap(def map[string]string) map[string]string
Scan(val interface{}) error
Bytes() []byte
}
type Options struct {
Loader loader.Loader
Reader reader.Reader
Source []source.Source
// for alternative data
Context context.Context
// Is the value being read a secret?
// If true, the Config will try to decode it with `SecretKey`
Secret bool
}
// Option sets values in Options
type Option func(o *Options)
// NewConfig returns new config
func NewConfig(opts ...Option) (Config, error) {
return newConfig(opts...)
func Secret(isSecret bool) Option {
return func(o *Options) {
o.Secret = isSecret
}
}

301
config/default.go
View File

@@ -1,301 +0,0 @@
package config
import (
"bytes"
"sync"
"time"
"github.com/micro/go-micro/v3/config/loader"
"github.com/micro/go-micro/v3/config/loader/memory"
"github.com/micro/go-micro/v3/config/reader"
"github.com/micro/go-micro/v3/config/reader/json"
"github.com/micro/go-micro/v3/config/source"
)
type config struct {
exit chan bool
opts Options
sync.RWMutex
// the current snapshot
snap *loader.Snapshot
// the current values
vals reader.Values
}
type watcher struct {
lw loader.Watcher
rd reader.Reader
path []string
value reader.Value
}
func newConfig(opts ...Option) (Config, error) {
var c config
if err := c.Init(opts...); err != nil {
return nil, err
}
go c.run()
return &c, nil
}
func (c *config) Init(opts ...Option) error {
c.opts = Options{
Reader: json.NewReader(),
}
c.exit = make(chan bool)
for _, o := range opts {
o(&c.opts)
}
// default loader uses the configured reader
if c.opts.Loader == nil {
c.opts.Loader = memory.NewLoader(memory.WithReader(c.opts.Reader))
}
err := c.opts.Loader.Load(c.opts.Source...)
if err != nil {
return err
}
c.snap, err = c.opts.Loader.Snapshot()
if err != nil {
return err
}
c.vals, err = c.opts.Reader.Values(c.snap.ChangeSet)
if err != nil {
return err
}
return nil
}
func (c *config) Options() Options {
return c.opts
}
func (c *config) run() {
watch := func(w loader.Watcher) error {
for {
// get changeset
snap, err := w.Next()
if err != nil {
return err
}
c.Lock()
if c.snap != nil && c.snap.Version >= snap.Version {
c.Unlock()
continue
}
// save
c.snap = snap
// set values
c.vals, _ = c.opts.Reader.Values(snap.ChangeSet)
c.Unlock()
}
}
for {
w, err := c.opts.Loader.Watch()
if err != nil {
time.Sleep(time.Second)
continue
}
done := make(chan bool)
// the stop watch func
go func() {
select {
case <-done:
case <-c.exit:
}
w.Stop()
}()
// block watch
if err := watch(w); err != nil {
// do something better
time.Sleep(time.Second)
}
// close done chan
close(done)
// if the config is closed exit
select {
case <-c.exit:
return
default:
}
}
}
func (c *config) Map() map[string]interface{} {
c.RLock()
defer c.RUnlock()
return c.vals.Map()
}
func (c *config) Scan(v interface{}) error {
c.RLock()
defer c.RUnlock()
return c.vals.Scan(v)
}
// sync loads all the sources, calls the parser and updates the config
func (c *config) Sync() error {
if err := c.opts.Loader.Sync(); err != nil {
return err
}
snap, err := c.opts.Loader.Snapshot()
if err != nil {
return err
}
c.Lock()
defer c.Unlock()
c.snap = snap
vals, err := c.opts.Reader.Values(snap.ChangeSet)
if err != nil {
return err
}
c.vals = vals
return nil
}
func (c *config) Close() error {
select {
case <-c.exit:
return nil
default:
close(c.exit)
}
return nil
}
func (c *config) Get(path ...string) reader.Value {
c.RLock()
defer c.RUnlock()
// did sync actually work?
if c.vals != nil {
return c.vals.Get(path...)
}
// no value
return newValue()
}
func (c *config) Set(val interface{}, path ...string) {
c.Lock()
defer c.Unlock()
if c.vals != nil {
c.vals.Set(val, path...)
}
return
}
func (c *config) Del(path ...string) {
c.Lock()
defer c.Unlock()
if c.vals != nil {
c.vals.Del(path...)
}
return
}
func (c *config) Bytes() []byte {
c.RLock()
defer c.RUnlock()
if c.vals == nil {
return []byte{}
}
return c.vals.Bytes()
}
func (c *config) Load(sources ...source.Source) error {
if err := c.opts.Loader.Load(sources...); err != nil {
return err
}
snap, err := c.opts.Loader.Snapshot()
if err != nil {
return err
}
c.Lock()
defer c.Unlock()
c.snap = snap
vals, err := c.opts.Reader.Values(snap.ChangeSet)
if err != nil {
return err
}
c.vals = vals
return nil
}
func (c *config) Watch(path ...string) (Watcher, error) {
value := c.Get(path...)
w, err := c.opts.Loader.Watch(path...)
if err != nil {
return nil, err
}
return &watcher{
lw: w,
rd: c.opts.Reader,
path: path,
value: value,
}, nil
}
func (c *config) String() string {
return "config"
}
func (w *watcher) Next() (reader.Value, error) {
for {
s, err := w.lw.Next()
if err != nil {
return nil, err
}
// only process changes
if bytes.Equal(w.value.Bytes(), s.ChangeSet.Data) {
continue
}
v, err := w.rd.Values(s.ChangeSet)
if err != nil {
return nil, err
}
w.value = v.Get()
return w.value, nil
}
}
func (w *watcher) Stop() error {
return w.lw.Stop()
}

166
config/default_test.go
View File

@@ -1,166 +0,0 @@
package config
import (
"fmt"
"os"
"path/filepath"
"runtime"
"strings"
"testing"
"time"
"github.com/micro/go-micro/v3/config/source"
"github.com/micro/go-micro/v3/config/source/env"
"github.com/micro/go-micro/v3/config/source/file"
"github.com/micro/go-micro/v3/config/source/memory"
)
func createFileForIssue18(t *testing.T, content string) *os.File {
data := []byte(content)
path := filepath.Join(os.TempDir(), fmt.Sprintf("file.%d", time.Now().UnixNano()))
fh, err := os.Create(path)
if err != nil {
t.Error(err)
}
_, err = fh.Write(data)
if err != nil {
t.Error(err)
}
return fh
}
func createFileForTest(t *testing.T) *os.File {
data := []byte(`{"foo": "bar"}`)
path := filepath.Join(os.TempDir(), fmt.Sprintf("file.%d", time.Now().UnixNano()))
fh, err := os.Create(path)
if err != nil {
t.Error(err)
}
_, err = fh.Write(data)
if err != nil {
t.Error(err)
}
return fh
}
func TestConfigLoadWithGoodFile(t *testing.T) {
fh := createFileForTest(t)
path := fh.Name()
defer func() {
fh.Close()
os.Remove(path)
}()
// Create new config
conf, err := NewConfig()
if err != nil {
t.Fatalf("Expected no error but got %v", err)
}
// Load file source
if err := conf.Load(file.NewSource(
file.WithPath(path),
)); err != nil {
t.Fatalf("Expected no error but got %v", err)
}
}
func TestConfigLoadWithInvalidFile(t *testing.T) {
fh := createFileForTest(t)
path := fh.Name()
defer func() {
fh.Close()
os.Remove(path)
}()
// Create new config
conf, err := NewConfig()
if err != nil {
t.Fatalf("Expected no error but got %v", err)
}
// Load file source
err = conf.Load(file.NewSource(
file.WithPath(path),
file.WithPath("/i/do/not/exists.json"),
))
if err == nil {
t.Fatal("Expected error but none !")
}
if !strings.Contains(fmt.Sprintf("%v", err), "/i/do/not/exists.json") {
t.Fatalf("Expected error to contain the unexisting file but got %v", err)
}
}
func TestConfigMerge(t *testing.T) {
fh := createFileForIssue18(t, `{
"amqp": {
"host": "rabbit.platform",
"port": 80
},
"handler": {
"exchange": "springCloudBus"
}
}`)
path := fh.Name()
defer func() {
fh.Close()
os.Remove(path)
}()
os.Setenv("AMQP_HOST", "rabbit.testing.com")
conf, err := NewConfig()
if err != nil {
t.Fatalf("Expected no error but got %v", err)
}
if err := conf.Load(
file.NewSource(
file.WithPath(path),
),
env.NewSource(),
); err != nil {
t.Fatalf("Expected no error but got %v", err)
}
actualHost := conf.Get("amqp", "host").String("backup")
if actualHost != "rabbit.testing.com" {
t.Fatalf("Expected %v but got %v",
"rabbit.testing.com",
actualHost)
}
}
func equalS(t *testing.T, actual, expect string) {
if actual != expect {
t.Errorf("Expected %s but got %s", actual, expect)
}
}
func TestConfigWatcherDirtyOverrite(t *testing.T) {
n := runtime.GOMAXPROCS(0)
defer runtime.GOMAXPROCS(n)
runtime.GOMAXPROCS(1)
l := 100
ss := make([]source.Source, l, l)
for i := 0; i < l; i++ {
ss[i] = memory.NewSource(memory.WithJSON([]byte(fmt.Sprintf(`{"key%d": "val%d"}`, i, i))))
}
conf, _ := NewConfig()
for _, s := range ss {
_ = conf.Load(s)
}
runtime.Gosched()
for i, _ := range ss {
k := fmt.Sprintf("key%d", i)
v := fmt.Sprintf("val%d", i)
equalS(t, conf.Get(k).String(""), v)
}
}

8
config/encoder/encoder.go
View File

@@ -1,8 +0,0 @@
// Package encoder handles source encoding formats
package encoder
type Encoder interface {
Encode(interface{}) ([]byte, error)
Decode([]byte, interface{}) error
String() string
}

26
config/encoder/hcl/hcl.go
View File

@@ -1,26 +0,0 @@
package hcl
import (
"encoding/json"
"github.com/hashicorp/hcl"
"github.com/micro/go-micro/v3/config/encoder"
)
type hclEncoder struct{}
func (h hclEncoder) Encode(v interface{}) ([]byte, error) {
return json.Marshal(v)
}
func (h hclEncoder) Decode(d []byte, v interface{}) error {
return hcl.Unmarshal(d, v)
}
func (h hclEncoder) String() string {
return "hcl"
}
func NewEncoder() encoder.Encoder {
return hclEncoder{}
}

25
config/encoder/json/json.go
View File

@@ -1,25 +0,0 @@
package json
import (
"encoding/json"
"github.com/micro/go-micro/v3/config/encoder"
)
type jsonEncoder struct{}
func (j jsonEncoder) Encode(v interface{}) ([]byte, error) {
return json.Marshal(v)
}
func (j jsonEncoder) Decode(d []byte, v interface{}) error {
return json.Unmarshal(d, v)
}
func (j jsonEncoder) String() string {
return "json"
}
func NewEncoder() encoder.Encoder {
return jsonEncoder{}
}

32
config/encoder/toml/toml.go
View File

@@ -1,32 +0,0 @@
package toml
import (
"bytes"
"github.com/BurntSushi/toml"
"github.com/micro/go-micro/v3/config/encoder"
)
type tomlEncoder struct{}
func (t tomlEncoder) Encode(v interface{}) ([]byte, error) {
b := bytes.NewBuffer(nil)
defer b.Reset()
err := toml.NewEncoder(b).Encode(v)
if err != nil {
return nil, err
}
return b.Bytes(), nil
}
func (t tomlEncoder) Decode(d []byte, v interface{}) error {
return toml.Unmarshal(d, v)
}
func (t tomlEncoder) String() string {
return "toml"
}
func NewEncoder() encoder.Encoder {
return tomlEncoder{}
}

25
config/encoder/xml/xml.go
View File

@@ -1,25 +0,0 @@
package xml
import (
"encoding/xml"
"github.com/micro/go-micro/v3/config/encoder"
)
type xmlEncoder struct{}
func (x xmlEncoder) Encode(v interface{}) ([]byte, error) {
return xml.Marshal(v)
}
func (x xmlEncoder) Decode(d []byte, v interface{}) error {
return xml.Unmarshal(d, v)
}
func (x xmlEncoder) String() string {
return "xml"
}
func NewEncoder() encoder.Encoder {
return xmlEncoder{}
}

24
config/encoder/yaml/yaml.go
View File

@@ -1,24 +0,0 @@
package yaml
import (
"github.com/ghodss/yaml"
"github.com/micro/go-micro/v3/config/encoder"
)
type yamlEncoder struct{}
func (y yamlEncoder) Encode(v interface{}) ([]byte, error) {
return yaml.Marshal(v)
}
func (y yamlEncoder) Decode(d []byte, v interface{}) error {
return yaml.Unmarshal(d, v)
}
func (y yamlEncoder) String() string {
return "yaml"
}
func NewEncoder() encoder.Encoder {
return yamlEncoder{}
}

63
config/loader/loader.go
View File

@@ -1,63 +0,0 @@
// package loader manages loading from multiple sources
package loader
import (
"context"
"github.com/micro/go-micro/v3/config/reader"
"github.com/micro/go-micro/v3/config/source"
)
// Loader manages loading sources
type Loader interface {
// Stop the loader
Close() error
// Load the sources
Load(...source.Source) error
// A Snapshot of loaded config
Snapshot() (*Snapshot, error)
// Force sync of sources
Sync() error
// Watch for changes
Watch(...string) (Watcher, error)
// Name of loader
String() string
}
// Watcher lets you watch sources and returns a merged ChangeSet
type Watcher interface {
// First call to next may return the current Snapshot
// If you are watching a path then only the data from
// that path is returned.
Next() (*Snapshot, error)
// Stop watching for changes
Stop() error
}
// Snapshot is a merged ChangeSet
type Snapshot struct {
// The merged ChangeSet
ChangeSet *source.ChangeSet
// Deterministic and comparable version of the snapshot
Version string
}
type Options struct {
Reader reader.Reader
Source []source.Source
// for alternative data
Context context.Context
}
type Option func(o *Options)
// Copy snapshot
func Copy(s *Snapshot) *Snapshot {
cs := *(s.ChangeSet)
return &Snapshot{
ChangeSet: &cs,
Version: s.Version,
}
}

459
config/loader/memory/memory.go
View File

@@ -1,459 +0,0 @@
package memory
import (
"bytes"
"container/list"
"errors"
"fmt"
"strings"
"sync"
"time"
"github.com/micro/go-micro/v3/config/loader"
"github.com/micro/go-micro/v3/config/reader"
"github.com/micro/go-micro/v3/config/reader/json"
"github.com/micro/go-micro/v3/config/source"
)
type memory struct {
exit chan bool
opts loader.Options
sync.RWMutex
// the current snapshot
snap *loader.Snapshot
// the current values
vals reader.Values
// all the sets
sets []*source.ChangeSet
// all the sources
sources []source.Source
watchers *list.List
}
type updateValue struct {
version string
value reader.Value
}
type watcher struct {
exit chan bool
path []string
value reader.Value
reader reader.Reader
version string
updates chan updateValue
}
func (m *memory) watch(idx int, s source.Source) {
// watches a source for changes
watch := func(idx int, s source.Watcher) error {
for {
// get changeset
cs, err := s.Next()
if err != nil {
return err
}
m.Lock()
// save
m.sets[idx] = cs
// merge sets
set, err := m.opts.Reader.Merge(m.sets...)
if err != nil {
m.Unlock()
return err
}
// set values
m.vals, _ = m.opts.Reader.Values(set)
m.snap = &loader.Snapshot{
ChangeSet: set,
Version: genVer(),
}
m.Unlock()
// send watch updates
m.update()
}
}
for {
// watch the source
w, err := s.Watch()
if err != nil {
time.Sleep(time.Second)
continue
}
done := make(chan bool)
// the stop watch func
go func() {
select {
case <-done:
case <-m.exit:
}
w.Stop()
}()
// block watch
if err := watch(idx, w); err != nil {
// do something better
time.Sleep(time.Second)
}
// close done chan
close(done)
// if the config is closed exit
select {
case <-m.exit:
return
default:
}
}
}
func (m *memory) loaded() bool {
var loaded bool
m.RLock()
if m.vals != nil {
loaded = true
}
m.RUnlock()
return loaded
}
// reload reads the sets and creates new values
func (m *memory) reload() error {
m.Lock()
// merge sets
set, err := m.opts.Reader.Merge(m.sets...)
if err != nil {
m.Unlock()
return err
}
// set values
m.vals, _ = m.opts.Reader.Values(set)
m.snap = &loader.Snapshot{
ChangeSet: set,
Version: genVer(),
}
m.Unlock()
// update watchers
m.update()
return nil
}
func (m *memory) update() {
watchers := make([]*watcher, 0, m.watchers.Len())
m.RLock()
for e := m.watchers.Front(); e != nil; e = e.Next() {
watchers = append(watchers, e.Value.(*watcher))
}
vals := m.vals
snap := m.snap
m.RUnlock()
for _, w := range watchers {
if w.version >= snap.Version {
continue
}
uv := updateValue{
version: m.snap.Version,
value: vals.Get(w.path...),
}
select {
case <-w.exit:
continue
default:
}
select {
case w.updates <- uv:
default:
}
}
}
// Snapshot returns a snapshot of the current loaded config
func (m *memory) Snapshot() (*loader.Snapshot, error) {
if m.loaded() {
m.RLock()
snap := loader.Copy(m.snap)
m.RUnlock()
return snap, nil
}
// not loaded, sync
if err := m.Sync(); err != nil {
return nil, err
}
// make copy
m.RLock()
snap := loader.Copy(m.snap)
m.RUnlock()
return snap, nil
}
// Sync loads all the sources, calls the parser and updates the config
func (m *memory) Sync() error {
//nolint:prealloc
var sets []*source.ChangeSet
m.Lock()
// read the source
var gerr []string
for _, source := range m.sources {
ch, err := source.Read()
if err != nil {
gerr = append(gerr, err.Error())
continue
}
sets = append(sets, ch)
}
// merge sets
set, err := m.opts.Reader.Merge(sets...)
if err != nil {
m.Unlock()
return err
}
// set values
vals, err := m.opts.Reader.Values(set)
if err != nil {
m.Unlock()
return err
}
m.vals = vals
m.snap = &loader.Snapshot{
ChangeSet: set,
Version: genVer(),
}
m.Unlock()
// update watchers
m.update()
if len(gerr) > 0 {
return fmt.Errorf("source loading errors: %s", strings.Join(gerr, "\n"))
}
return nil
}
func (m *memory) Close() error {
select {
case <-m.exit:
return nil
default:
close(m.exit)
}
return nil
}
func (m *memory) Get(path ...string) (reader.Value, error) {
if !m.loaded() {
if err := m.Sync(); err != nil {
return nil, err
}
}
m.Lock()
defer m.Unlock()
// did sync actually work?
if m.vals != nil {
return m.vals.Get(path...), nil
}
// assuming vals is nil
// create new vals
ch := m.snap.ChangeSet
// we are truly screwed, trying to load in a hacked way
v, err := m.opts.Reader.Values(ch)
if err != nil {
return nil, err
}
// lets set it just because
m.vals = v
if m.vals != nil {
return m.vals.Get(path...), nil
}
// ok we're going hardcore now
return nil, errors.New("no values")
}
func (m *memory) Load(sources ...source.Source) error {
var gerrors []string
for _, source := range sources {
set, err := source.Read()
if err != nil {
gerrors = append(gerrors,
fmt.Sprintf("error loading source %s: %v",
source,
err))
// continue processing
continue
}
m.Lock()
m.sources = append(m.sources, source)
m.sets = append(m.sets, set)
idx := len(m.sets) - 1
m.Unlock()
go m.watch(idx, source)
}
if err := m.reload(); err != nil {
gerrors = append(gerrors, err.Error())
}
// Return errors
if len(gerrors) != 0 {
return errors.New(strings.Join(gerrors, "\n"))
}
return nil
}
func (m *memory) Watch(path ...string) (loader.Watcher, error) {
value, err := m.Get(path...)
if err != nil {
return nil, err
}
m.Lock()
w := &watcher{
exit: make(chan bool),
path: path,
value: value,
reader: m.opts.Reader,
updates: make(chan updateValue, 1),
version: m.snap.Version,
}
e := m.watchers.PushBack(w)
m.Unlock()
go func() {
<-w.exit
m.Lock()
m.watchers.Remove(e)
m.Unlock()
}()
return w, nil
}
func (m *memory) String() string {
return "memory"
}
func (w *watcher) Next() (*loader.Snapshot, error) {
update := func(v reader.Value) *loader.Snapshot {
w.value = v
cs := &source.ChangeSet{
Data: v.Bytes(),
Format: w.reader.String(),
Source: "memory",
Timestamp: time.Now(),
}
cs.Checksum = cs.Sum()
return &loader.Snapshot{
ChangeSet: cs,
Version: w.version,
}
}
for {
select {
case <-w.exit:
return nil, errors.New("watcher stopped")
case uv := <-w.updates:
if uv.version <= w.version {
continue
}
v := uv.value
w.version = uv.version
if bytes.Equal(w.value.Bytes(), v.Bytes()) {
continue
}
return update(v), nil
}
}
}
func (w *watcher) Stop() error {
select {
case <-w.exit:
default:
close(w.exit)
close(w.updates)
}
return nil
}
func genVer() string {
return fmt.Sprintf("%d", time.Now().UnixNano())
}
func NewLoader(opts ...loader.Option) loader.Loader {
options := loader.Options{
Reader: json.NewReader(),
}
for _, o := range opts {
o(&options)
}
m := &memory{
exit: make(chan bool),
opts: options,
watchers: list.New(),
sources: options.Source,
}
m.sets = make([]*source.ChangeSet, len(options.Source))
for i, s := range options.Source {
m.sets[i] = &source.ChangeSet{Source: s.String()}
go m.watch(i, s)
}
return m
}

21
config/loader/memory/options.go
View File

@@ -1,21 +0,0 @@
package memory
import (
"github.com/micro/go-micro/v3/config/loader"
"github.com/micro/go-micro/v3/config/reader"
"github.com/micro/go-micro/v3/config/source"
)
// WithSource appends a source to list of sources
func WithSource(s source.Source) loader.Option {
return func(o *loader.Options) {
o.Source = append(o.Source, s)
}
}
// WithReader sets the config reader
func WithReader(r reader.Reader) loader.Option {
return func(o *loader.Options) {
o.Reader = r
}
}

28
config/options.go
View File

@@ -1,28 +0,0 @@
package config
import (
"github.com/micro/go-micro/v3/config/loader"
"github.com/micro/go-micro/v3/config/reader"
"github.com/micro/go-micro/v3/config/source"
)
// WithLoader sets the loader for manager config
func WithLoader(l loader.Loader) Option {
return func(o *Options) {
o.Loader = l
}
}
// WithSource appends a source to list of sources
func WithSource(s source.Source) Option {
return func(o *Options) {
o.Source = append(o.Source, s)
}
}
// WithReader sets the config reader
func WithReader(r reader.Reader) Option {
return func(o *Options) {
o.Reader = r
}
}

83
config/reader/json/json.go
View File

@@ -1,83 +0,0 @@
package json
import (
"errors"
"time"
"github.com/imdario/mergo"
"github.com/micro/go-micro/v3/config/encoder"
"github.com/micro/go-micro/v3/config/encoder/json"
"github.com/micro/go-micro/v3/config/reader"
"github.com/micro/go-micro/v3/config/source"
)
type jsonReader struct {
opts reader.Options
json encoder.Encoder
}
func (j *jsonReader) Merge(changes ...*source.ChangeSet) (*source.ChangeSet, error) {
var merged map[string]interface{}
for _, m := range changes {
if m == nil {
continue
}
if len(m.Data) == 0 {
continue
}
codec, ok := j.opts.Encoding[m.Format]
if !ok {
// fallback
codec = j.json
}
var data map[string]interface{}
if err := codec.Decode(m.Data, &data); err != nil {
return nil, err
}
if err := mergo.Map(&merged, data, mergo.WithOverride); err != nil {
return nil, err
}
}
b, err := j.json.Encode(merged)
if err != nil {
return nil, err
}
cs := &source.ChangeSet{
Timestamp: time.Now(),
Data: b,
Source: "json",
Format: j.json.String(),
}
cs.Checksum = cs.Sum()
return cs, nil
}
func (j *jsonReader) Values(ch *source.ChangeSet) (reader.Values, error) {
if ch == nil {
return nil, errors.New("changeset is nil")
}
if ch.Format != "json" {
return nil, errors.New("unsupported format")
}
return newValues(ch, j.opts)
}
func (j *jsonReader) String() string {
return "json"
}
// NewReader creates a json reader
func NewReader(opts ...reader.Option) reader.Reader {
options := reader.NewOptions(opts...)
return &jsonReader{
json: json.NewEncoder(),
opts: options,
}
}

79
config/reader/json/json_test.go
View File

@@ -1,79 +0,0 @@
package json
import (
"testing"
"github.com/micro/go-micro/v3/config/reader"
"github.com/micro/go-micro/v3/config/source"
)
func TestReader(t *testing.T) {
data := []byte(`{"foo": "bar", "baz": {"bar": "cat"}}`)
testData := []struct {
path []string
value string
}{
{
[]string{"foo"},
"bar",
},
{
[]string{"baz", "bar"},
"cat",
},
}
values := newTestValues(t, data)
for _, test := range testData {
if v := values.Get(test.path...).String(""); v != test.value {
t.Fatalf("Expected %s got %s for path %v", test.value, v, test.path)
}
}
}
func TestDisableReplaceEnvVars(t *testing.T) {
data := []byte(`{"foo": "bar", "baz": {"bar": "test/${test}"}}`)
tests := []struct {
path []string
value string
opts []reader.Option
}{
{
[]string{"baz", "bar"},
"test/",
nil,
},
{
[]string{"baz", "bar"},
"test/${test}",
[]reader.Option{reader.WithDisableReplaceEnvVars()},
},
}
for _, test := range tests {
values := newTestValues(t, data, test.opts...)
if v := values.Get(test.path...).String(""); v != test.value {
t.Fatalf("Expected %s got %s for path %v", test.value, v, test.path)
}
}
}
func newTestValues(t *testing.T, data []byte, opts ...reader.Option) reader.Values {
r := NewReader(opts...)
c, err := r.Merge(&source.ChangeSet{Data: data}, &source.ChangeSet{})
if err != nil {
t.Fatal(err)
}
values, err := r.Values(c)
if err != nil {
t.Fatal(err)
}
return values
}

206
config/reader/json/values.go
View File

@@ -1,206 +0,0 @@
package json
import (
"encoding/json"
"fmt"
"strconv"
"strings"
"time"
simple "github.com/bitly/go-simplejson"
"github.com/micro/go-micro/v3/config/reader"
"github.com/micro/go-micro/v3/config/source"
)
type jsonValues struct {
ch *source.ChangeSet
sj *simple.Json
}
type jsonValue struct {
*simple.Json
}
func newValues(ch *source.ChangeSet, opts reader.Options) (reader.Values, error) {
sj := simple.New()
data := ch.Data
if !opts.DisableReplaceEnvVars {
data, _ = reader.ReplaceEnvVars(ch.Data)
}
if err := sj.UnmarshalJSON(data); err != nil {
sj.SetPath(nil, string(ch.Data))
}
return &jsonValues{ch, sj}, nil
}
func (j *jsonValues) Get(path ...string) reader.Value {
return &jsonValue{j.sj.GetPath(path...)}
}
func (j *jsonValues) Del(path ...string) {
// delete the tree?
if len(path) == 0 {
j.sj = simple.New()
return
}
if len(path) == 1 {
j.sj.Del(path[0])
return
}
vals := j.sj.GetPath(path[:len(path)-1]...)
vals.Del(path[len(path)-1])
j.sj.SetPath(path[:len(path)-1], vals.Interface())
return
}
func (j *jsonValues) Set(val interface{}, path ...string) {
j.sj.SetPath(path, val)
}
func (j *jsonValues) Bytes() []byte {
b, _ := j.sj.MarshalJSON()
return b
}
func (j *jsonValues) Map() map[string]interface{} {
m, _ := j.sj.Map()
return m
}
func (j *jsonValues) Scan(v interface{}) error {
b, err := j.sj.MarshalJSON()
if err != nil {
return err
}
return json.Unmarshal(b, v)
}
func (j *jsonValues) String() string {
return "json"
}
func (j *jsonValue) Bool(def bool) bool {
b, err := j.Json.Bool()
if err == nil {
return b
}
str, ok := j.Interface().(string)
if !ok {
return def
}
b, err = strconv.ParseBool(str)
if err != nil {
return def
}
return b
}
func (j *jsonValue) Int(def int) int {
i, err := j.Json.Int()
if err == nil {
return i
}
str, ok := j.Interface().(string)
if !ok {
return def
}
i, err = strconv.Atoi(str)
if err != nil {
return def
}
return i
}
func (j *jsonValue) String(def string) string {
return j.Json.MustString(def)
}
func (j *jsonValue) Float64(def float64) float64 {
f, err := j.Json.Float64()
if err == nil {
return f
}
str, ok := j.Interface().(string)
if !ok {
return def
}
f, err = strconv.ParseFloat(str, 64)
if err != nil {
return def
}
return f
}
func (j *jsonValue) Duration(def time.Duration) time.Duration {
v, err := j.Json.String()
if err != nil {
return def
}
value, err := time.ParseDuration(v)
if err != nil {
return def
}
return value
}
func (j *jsonValue) StringSlice(def []string) []string {
v, err := j.Json.String()
if err == nil {
sl := strings.Split(v, ",")
if len(sl) > 1 {
return sl
}
}
return j.Json.MustStringArray(def)
}
func (j *jsonValue) StringMap(def map[string]string) map[string]string {
m, err := j.Json.Map()
if err != nil {
return def
}
res := map[string]string{}
for k, v := range m {
res[k] = fmt.Sprintf("%v", v)
}
return res
}
func (j *jsonValue) Scan(v interface{}) error {
b, err := j.Json.MarshalJSON()
if err != nil {
return err
}
return json.Unmarshal(b, v)
}
func (j *jsonValue) Bytes() []byte {
b, err := j.Json.Bytes()
if err != nil {
// try return marshalled
b, err = j.Json.MarshalJSON()
if err != nil {
return []byte{}
}
return b
}
return b
}

86
config/reader/json/values_test.go
View File

@@ -1,86 +0,0 @@
package json
import (
"reflect"
"testing"
"github.com/micro/go-micro/v3/config/reader"
"github.com/micro/go-micro/v3/config/source"
)
func TestValues(t *testing.T) {
emptyStr := ""
testData := []struct {
csdata []byte
path []string
accepter interface{}
value interface{}
}{
{
[]byte(`{"foo": "bar", "baz": {"bar": "cat"}}`),
[]string{"foo"},
emptyStr,
"bar",
},
{
[]byte(`{"foo": "bar", "baz": {"bar": "cat"}}`),
[]string{"baz", "bar"},
emptyStr,
"cat",
},
}
for idx, test := range testData {
values, err := newValues(&source.ChangeSet{
Data: test.csdata,
}, reader.Options{})
if err != nil {
t.Fatal(err)
}
err = values.Get(test.path...).Scan(&test.accepter)
if err != nil {
t.Fatal(err)
}
if test.accepter != test.value {
t.Fatalf("No.%d Expected %v got %v for path %v", idx, test.value, test.accepter, test.path)
}
}
}
func TestStructArray(t *testing.T) {
type T struct {
Foo string
}
emptyTSlice := []T{}
testData := []struct {
csdata []byte
accepter []T
value []T
}{
{
[]byte(`[{"foo": "bar"}]`),
emptyTSlice,
[]T{{Foo: "bar"}},
},
}
for idx, test := range testData {
values, err := newValues(&source.ChangeSet{
Data: test.csdata,
}, reader.Options{})
if err != nil {
t.Fatal(err)
}
err = values.Get().Scan(&test.accepter)
if err != nil {
t.Fatal(err)
}
if !reflect.DeepEqual(test.accepter, test.value) {
t.Fatalf("No.%d Expected %v got %v", idx, test.value, test.accepter)
}
}
}

50
config/reader/options.go
View File

@@ -1,50 +0,0 @@
package reader
import (
"github.com/micro/go-micro/v3/config/encoder"
"github.com/micro/go-micro/v3/config/encoder/hcl"
"github.com/micro/go-micro/v3/config/encoder/json"
"github.com/micro/go-micro/v3/config/encoder/toml"
"github.com/micro/go-micro/v3/config/encoder/xml"
"github.com/micro/go-micro/v3/config/encoder/yaml"
)
type Options struct {
Encoding map[string]encoder.Encoder
DisableReplaceEnvVars bool
}
type Option func(o *Options)
func NewOptions(opts ...Option) Options {
options := Options{
Encoding: map[string]encoder.Encoder{
"json": json.NewEncoder(),
"yaml": yaml.NewEncoder(),
"toml": toml.NewEncoder(),
"xml": xml.NewEncoder(),
"hcl": hcl.NewEncoder(),
"yml": yaml.NewEncoder(),
},
}
for _, o := range opts {
o(&options)
}
return options
}
func WithEncoder(e encoder.Encoder) Option {
return func(o *Options) {
if o.Encoding == nil {
o.Encoding = make(map[string]encoder.Encoder)
}
o.Encoding[e.String()] = e
}
}
// WithDisableReplaceEnvVars disables the environment variable interpolation preprocessor
func WithDisableReplaceEnvVars() Option {
return func(o *Options) {
o.DisableReplaceEnvVars = true
}
}

23
config/reader/preprocessor.go
View File

@@ -1,23 +0,0 @@
package reader
import (
"os"
"regexp"
)
func ReplaceEnvVars(raw []byte) ([]byte, error) {
re := regexp.MustCompile(`\$\{([A-Za-z0-9_]+)\}`)
if re.Match(raw) {
dataS := string(raw)
res := re.ReplaceAllStringFunc(dataS, replaceEnvVars)
return []byte(res), nil
} else {
return raw, nil
}
}
func replaceEnvVars(element string) string {
v := element[2 : len(element)-1]
el := os.Getenv(v)
return el
}

73
config/reader/preprocessor_test.go
View File

@@ -1,73 +0,0 @@
package reader
import (
"os"
"strings"
"testing"
)
func TestReplaceEnvVars(t *testing.T) {
os.Setenv("myBar", "cat")
os.Setenv("MYBAR", "cat")
os.Setenv("my_Bar", "cat")
os.Setenv("myBar_", "cat")
testData := []struct {
expected string
data []byte
}{
// Right use cases
{
`{"foo": "bar", "baz": {"bar": "cat"}}`,
[]byte(`{"foo": "bar", "baz": {"bar": "${myBar}"}}`),
},
{
`{"foo": "bar", "baz": {"bar": "cat"}}`,
[]byte(`{"foo": "bar", "baz": {"bar": "${MYBAR}"}}`),
},
{
`{"foo": "bar", "baz": {"bar": "cat"}}`,
[]byte(`{"foo": "bar", "baz": {"bar": "${my_Bar}"}}`),
},
{
`{"foo": "bar", "baz": {"bar": "cat"}}`,
[]byte(`{"foo": "bar", "baz": {"bar": "${myBar_}"}}`),
},
// Wrong use cases
{
`{"foo": "bar", "baz": {"bar": "${myBar-}"}}`,
[]byte(`{"foo": "bar", "baz": {"bar": "${myBar-}"}}`),
},
{
`{"foo": "bar", "baz": {"bar": "${}"}}`,
[]byte(`{"foo": "bar", "baz": {"bar": "${}"}}`),
},
{
`{"foo": "bar", "baz": {"bar": "$sss}"}}`,
[]byte(`{"foo": "bar", "baz": {"bar": "$sss}"}}`),
},
{
`{"foo": "bar", "baz": {"bar": "${sss"}}`,
[]byte(`{"foo": "bar", "baz": {"bar": "${sss"}}`),
},
{
`{"foo": "bar", "baz": {"bar": "{something}"}}`,
[]byte(`{"foo": "bar", "baz": {"bar": "{something}"}}`),
},
// Use cases without replace env vars
{
`{"foo": "bar", "baz": {"bar": "cat"}}`,
[]byte(`{"foo": "bar", "baz": {"bar": "cat"}}`),
},
}
for _, test := range testData {
res, err := ReplaceEnvVars(test.data)
if err != nil {
t.Fatal(err)
}
if strings.Compare(test.expected, string(res)) != 0 {
t.Fatalf("Expected %s got %s", test.expected, res)
}
}
}

38
config/reader/reader.go
View File

@@ -1,38 +0,0 @@
// Package reader parses change sets and provides config values
package reader
import (
"time"
"github.com/micro/go-micro/v3/config/source"
)
// Reader is an interface for merging changesets
type Reader interface {
Merge(...*source.ChangeSet) (*source.ChangeSet, error)
Values(*source.ChangeSet) (Values, error)
String() string
}
// Values is returned by the reader
type Values interface {
Bytes() []byte
Get(path ...string) Value
Set(val interface{}, path ...string)
Del(path ...string)
Map() map[string]interface{}
Scan(v interface{}) error
}
// Value represents a value of any type
type Value interface {
Bool(def bool) bool
Int(def int) int
String(def string) string
Float64(def float64) float64
Duration(def time.Duration) time.Duration
StringSlice(def []string) []string
StringMap(def map[string]string) map[string]string
Scan(val interface{}) error
Bytes() []byte
}

13
config/source/changeset.go
View File

@@ -1,13 +0,0 @@
package source
import (
"crypto/md5"
"fmt"
)
// Sum returns the md5 checksum of the ChangeSet data
func (c *ChangeSet) Sum() string {
h := md5.New()
h.Write(c.Data)
return fmt.Sprintf("%x", h.Sum(nil))
}

96
config/source/env/README.md vendored
View File

@@ -1,96 +0,0 @@
# Env Source
The env source reads config from environment variables
## Format
We expect environment variables to be in the standard format of FOO=bar
Keys are converted to lowercase and split on underscore.
### Example
```
DATABASE_ADDRESS=127.0.0.1
DATABASE_PORT=3306
```
Becomes
```json
{
"database": {
"address": "127.0.0.1",
"port": 3306
}
}
```
## Prefixes
Environment variables can be namespaced so we only have access to a subset. Two options are available:
```
WithPrefix(p ...string)
WithStrippedPrefix(p ...string)
```
The former will preserve the prefix and make it a top level key in the config. The latter eliminates the prefix, reducing the nesting by one.
#### Example:
Given ENVs of:
```
APP_DATABASE_ADDRESS=127.0.0.1
APP_DATABASE_PORT=3306
VAULT_ADDR=vault:1337
```
and a source initialized as follows:
```
src := env.NewSource(
env.WithPrefix("VAULT"),
env.WithStrippedPrefix("APP"),
)
```
The resulting config will be:
```
{
"database": {
"address": "127.0.0.1",
"port": 3306
},
"vault": {
"addr": "vault:1337"
}
}
```
## New Source
Specify source with data
```go
src := env.NewSource(
// optionally specify prefix
env.WithPrefix("MICRO"),
)
```
## Load Source
Load the source into config
```go
// Create new config
conf := config.NewConfig()
// Load env source
conf.Load(src)
```

146
config/source/env/env.go vendored
View File

@@ -1,146 +0,0 @@
package env
import (
"os"
"strconv"
"strings"
"time"
"github.com/imdario/mergo"
"github.com/micro/go-micro/v3/config/source"
)
var (
DefaultPrefixes = []string{}
)
type env struct {
prefixes []string
strippedPrefixes []string
opts source.Options
}
func (e *env) Read() (*source.ChangeSet, error) {
var changes map[string]interface{}
for _, env := range os.Environ() {
if len(e.prefixes) > 0 || len(e.strippedPrefixes) > 0 {
notFound := true
if _, ok := matchPrefix(e.prefixes, env); ok {
notFound = false
}
if match, ok := matchPrefix(e.strippedPrefixes, env); ok {
env = strings.TrimPrefix(env, match)
notFound = false
}
if notFound {
continue
}
}
pair := strings.SplitN(env, "=", 2)
value := pair[1]
keys := strings.Split(strings.ToLower(pair[0]), "_")
reverse(keys)
tmp := make(map[string]interface{})
for i, k := range keys {
if i == 0 {
if intValue, err := strconv.Atoi(value); err == nil {
tmp[k] = intValue
} else if boolValue, err := strconv.ParseBool(value); err == nil {
tmp[k] = boolValue
} else {
tmp[k] = value
}
continue
}
tmp = map[string]interface{}{k: tmp}
}
if err := mergo.Map(&changes, tmp); err != nil {
return nil, err
}
}
b, err := e.opts.Encoder.Encode(changes)
if err != nil {
return nil, err
}
cs := &source.ChangeSet{
Format: e.opts.Encoder.String(),
Data: b,
Timestamp: time.Now(),
Source: e.String(),
}
cs.Checksum = cs.Sum()
return cs, nil
}
func matchPrefix(pre []string, s string) (string, bool) {
for _, p := range pre {
if strings.HasPrefix(s, p) {
return p, true
}
}
return "", false
}
func reverse(ss []string) {
for i := len(ss)/2 - 1; i >= 0; i-- {
opp := len(ss) - 1 - i
ss[i], ss[opp] = ss[opp], ss[i]
}
}
func (e *env) Watch() (source.Watcher, error) {
return newWatcher()
}
func (e *env) Write(cs *source.ChangeSet) error {
return nil
}
func (e *env) String() string {
return "env"
}
// NewSource returns a config source for parsing ENV variables.
// Underscores are delimiters for nesting, and all keys are lowercased.
//
// Example:
// "DATABASE_SERVER_HOST=localhost" will convert to
//
// {
// "database": {
// "server": {
// "host": "localhost"
// }
// }
// }
func NewSource(opts ...source.Option) source.Source {
options := source.NewOptions(opts...)
var sp []string
var pre []string
if p, ok := options.Context.Value(strippedPrefixKey{}).([]string); ok {
sp = p
}
if p, ok := options.Context.Value(prefixKey{}).([]string); ok {
pre = p
}
if len(sp) > 0 || len(pre) > 0 {
pre = append(pre, DefaultPrefixes...)
}
return &env{prefixes: pre, strippedPrefixes: sp, opts: options}
}

112
config/source/env/env_test.go vendored
View File

@@ -1,112 +0,0 @@
package env
import (
"encoding/json"
"os"
"testing"
"time"
"github.com/micro/go-micro/v3/config/source"
)
func TestEnv_Read(t *testing.T) {
expected := map[string]map[string]string{
"database": {
"host": "localhost",
"password": "password",
"datasource": "user:password@tcp(localhost:port)/db?charset=utf8mb4&parseTime=True&loc=Local",
},
}
os.Setenv("DATABASE_HOST", "localhost")
os.Setenv("DATABASE_PASSWORD", "password")
os.Setenv("DATABASE_DATASOURCE", "user:password@tcp(localhost:port)/db?charset=utf8mb4&parseTime=True&loc=Local")
source := NewSource()
c, err := source.Read()
if err != nil {
t.Error(err)
}
var actual map[string]interface{}
if err := json.Unmarshal(c.Data, &actual); err != nil {
t.Error(err)
}
actualDB := actual["database"].(map[string]interface{})
for k, v := range expected["database"] {
a := actualDB[k]
if a != v {
t.Errorf("expected %v got %v", v, a)
}
}
}
func TestEnvvar_Prefixes(t *testing.T) {
os.Setenv("APP_DATABASE_HOST", "localhost")
os.Setenv("APP_DATABASE_PASSWORD", "password")
os.Setenv("VAULT_ADDR", "vault:1337")
os.Setenv("MICRO_REGISTRY", "mdns")
var prefixtests = []struct {
prefixOpts []source.Option
expectedKeys []string
}{
{[]source.Option{WithPrefix("APP", "MICRO")}, []string{"app", "micro"}},
{[]source.Option{WithPrefix("MICRO"), WithStrippedPrefix("APP")}, []string{"database", "micro"}},
{[]source.Option{WithPrefix("MICRO"), WithStrippedPrefix("APP")}, []string{"database", "micro"}},
}
for _, pt := range prefixtests {
source := NewSource(pt.prefixOpts...)
c, err := source.Read()
if err != nil {
t.Error(err)
}
var actual map[string]interface{}
if err := json.Unmarshal(c.Data, &actual); err != nil {
t.Error(err)
}
// assert other prefixes ignored
if l := len(actual); l != len(pt.expectedKeys) {
t.Errorf("expected %v top keys, got %v", len(pt.expectedKeys), l)
}
for _, k := range pt.expectedKeys {
if !containsKey(actual, k) {
t.Errorf("expected key %v, not found", k)
}
}
}
}
func TestEnvvar_WatchNextNoOpsUntilStop(t *testing.T) {
src := NewSource(WithStrippedPrefix("GOMICRO_"))
w, err := src.Watch()
if err != nil {
t.Error(err)
}
go func() {
time.Sleep(50 * time.Millisecond)
w.Stop()
}()
if _, err := w.Next(); err != source.ErrWatcherStopped {
t.Errorf("expected watcher stopped error, got %v", err)
}
}
func containsKey(m map[string]interface{}, s string) bool {
for k := range m {
if k == s {
return true
}
}
return false
}

50
config/source/env/options.go vendored
View File

@@ -1,50 +0,0 @@
package env
import (
"context"
"strings"
"github.com/micro/go-micro/v3/config/source"
)
type strippedPrefixKey struct{}
type prefixKey struct{}
// WithStrippedPrefix sets the environment variable prefixes to scope to.
// These prefixes will be removed from the actual config entries.
func WithStrippedPrefix(p ...string) source.Option {
return func(o *source.Options) {
if o.Context == nil {
o.Context = context.Background()
}
o.Context = context.WithValue(o.Context, strippedPrefixKey{}, appendUnderscore(p))
}
}
// WithPrefix sets the environment variable prefixes to scope to.
// These prefixes will not be removed. Each prefix will be considered a top level config entry.
func WithPrefix(p ...string) source.Option {
return func(o *source.Options) {
if o.Context == nil {
o.Context = context.Background()
}
o.Context = context.WithValue(o.Context, prefixKey{}, appendUnderscore(p))
}
}
func appendUnderscore(prefixes []string) []string {
//nolint:prealloc
var result []string
for _, p := range prefixes {
if !strings.HasSuffix(p, "_") {
result = append(result, p+"_")
continue
}
result = append(result, p)
}
return result
}

24
config/source/env/watcher.go vendored
View File

@@ -1,24 +0,0 @@
package env
import (
"github.com/micro/go-micro/v3/config/source"
)
type watcher struct {
exit chan struct{}
}
func (w *watcher) Next() (*source.ChangeSet, error) {
<-w.exit
return nil, source.ErrWatcherStopped
}
func (w *watcher) Stop() error {
close(w.exit)
return nil
}
func newWatcher() (source.Watcher, error) {
return &watcher{exit: make(chan struct{})}, nil
}

70
config/source/file/README.md
View File

@@ -1,70 +0,0 @@
# File Source
The file source reads config from a file.
It uses the File extension to determine the Format e.g `config.yaml` has the yaml format.
It does not make use of encoders or interpet the file data. If a file extension is not present
the source Format will default to the Encoder in options.
## Example
A config file format in json
```json
{
"hosts": {
"database": {
"address": "10.0.0.1",
"port": 3306
},
"cache": {
"address": "10.0.0.2",
"port": 6379
}
}
}
```
## New Source
Specify file source with path to file. Path is optional and will default to `config.json`
```go
fileSource := file.NewSource(
file.WithPath("/tmp/config.json"),
)
```
## File Format
To load different file formats e.g yaml, toml, xml simply specify them with their extension
```
fileSource := file.NewSource(
file.WithPath("/tmp/config.yaml"),
)
```
If you want to specify a file without extension, ensure you set the encoder to the same format
```
e := toml.NewEncoder()
fileSource := file.NewSource(
file.WithPath("/tmp/config"),
source.WithEncoder(e),
)
```
## Load Source
Load the source into config
```go
// Create new config
conf := config.NewConfig()
// Load file source
conf.Load(fileSource)
```

70
config/source/file/file.go
View File

@@ -1,70 +0,0 @@
// Package file is a file source. Expected format is json
package file
import (
"io/ioutil"
"os"
"github.com/micro/go-micro/v3/config/source"
)
type file struct {
path string
data []byte
opts source.Options
}
var (
DefaultPath = "config.json"
)
func (f *file) Read() (*source.ChangeSet, error) {
fh, err := os.Open(f.path)
if err != nil {
return nil, err
}
defer fh.Close()
b, err := ioutil.ReadAll(fh)
if err != nil {
return nil, err
}
info, err := fh.Stat()
if err != nil {
return nil, err
}
cs := &source.ChangeSet{
Format: format(f.path, f.opts.Encoder),
Source: f.String(),
Timestamp: info.ModTime(),
Data: b,
}
cs.Checksum = cs.Sum()
return cs, nil
}
func (f *file) String() string {
return "file"
}
func (f *file) Watch() (source.Watcher, error) {
if _, err := os.Stat(f.path); err != nil {
return nil, err
}
return newWatcher(f)
}
func (f *file) Write(cs *source.ChangeSet) error {
return nil
}
func NewSource(opts ...source.Option) source.Source {
options := source.NewOptions(opts...)
path := DefaultPath
f, ok := options.Context.Value(filePathKey{}).(string)
if ok {
path = f
}
return &file{opts: options, path: path}
}

66
config/source/file/file_test.go
View File

@@ -1,66 +0,0 @@
package file_test
import (
"fmt"
"os"
"path/filepath"
"testing"
"time"
"github.com/micro/go-micro/v3/config"
"github.com/micro/go-micro/v3/config/source/file"
)
func TestConfig(t *testing.T) {
data := []byte(`{"foo": "bar"}`)
path := filepath.Join(os.TempDir(), fmt.Sprintf("file.%d", time.Now().UnixNano()))
fh, err := os.Create(path)
if err != nil {
t.Error(err)
}
defer func() {
fh.Close()
os.Remove(path)
}()
_, err = fh.Write(data)
if err != nil {
t.Error(err)
}
conf, err := config.NewConfig()
if err != nil {
t.Fatal(err)
}
conf.Load(file.NewSource(file.WithPath(path)))
// simulate multiple close
go conf.Close()
go conf.Close()
}
func TestFile(t *testing.T) {
data := []byte(`{"foo": "bar"}`)
path := filepath.Join(os.TempDir(), fmt.Sprintf("file.%d", time.Now().UnixNano()))
fh, err := os.Create(path)
if err != nil {
t.Error(err)
}
defer func() {
fh.Close()
os.Remove(path)
}()
_, err = fh.Write(data)
if err != nil {
t.Error(err)
}
f := file.NewSource(file.WithPath(path))
c, err := f.Read()
if err != nil {
t.Error(err)
}
if string(c.Data) != string(data) {
t.Logf("%+v", c)
t.Error("data from file does not match")
}
}

15
config/source/file/format.go
View File

@@ -1,15 +0,0 @@
package file
import (
"strings"
"github.com/micro/go-micro/v3/config/encoder"
)
func format(p string, e encoder.Encoder) string {
parts := strings.Split(p, ".")
if len(parts) > 1 {
return parts[len(parts)-1]
}
return e.String()
}

31
config/source/file/format_test.go
View File

@@ -1,31 +0,0 @@
package file
import (
"testing"
"github.com/micro/go-micro/v3/config/source"
)
func TestFormat(t *testing.T) {
opts := source.NewOptions()
e := opts.Encoder
testCases := []struct {
p string
f string
}{
{"/foo/bar.json", "json"},
{"/foo/bar.yaml", "yaml"},
{"/foo/bar.xml", "xml"},
{"/foo/bar.conf.ini", "ini"},
{"conf", e.String()},
}
for _, d := range testCases {
f := format(d.p, e)
if f != d.f {
t.Fatalf("%s: expected %s got %s", d.p, d.f, f)
}
}
}

19
config/source/file/options.go
View File

@@ -1,19 +0,0 @@
package file
import (
"context"
"github.com/micro/go-micro/v3/config/source"
)
type filePathKey struct{}
// WithPath sets the path to file
func WithPath(p string) source.Option {
return func(o *source.Options) {
if o.Context == nil {
o.Context = context.Background()
}
o.Context = context.WithValue(o.Context, filePathKey{}, p)
}
}

77
config/source/file/watcher.go
View File

@@ -1,77 +0,0 @@
//+build !linux
package file
import (
"os"
"github.com/fsnotify/fsnotify"
"github.com/micro/go-micro/v3/config/source"
)
type watcher struct {
f *file
fw *fsnotify.Watcher
exit chan bool
}
func newWatcher(f *file) (source.Watcher, error) {
fw, err := fsnotify.NewWatcher()
if err != nil {
return nil, err
}
fw.Add(f.path)
return &watcher{
f: f,
fw: fw,
exit: make(chan bool),
}, nil
}
func (w *watcher) Next() (*source.ChangeSet, error) {
// is it closed?
select {
case <-w.exit:
return nil, source.ErrWatcherStopped
default:
}
for {
// try get the event
select {
case event, _ := <-w.fw.Events:
if event.Op == fsnotify.Rename {
// check existence of file, and add watch again
_, err := os.Stat(event.Name)
if err == nil || os.IsExist(err) {
w.fw.Add(event.Name)
}
}
// ARCH: Darwin Kernel Version 18.7.0
// ioutil.WriteFile truncates it before writing, but the problem is that
// you will receive two events(fsnotify.Chmod and fsnotify.Write).
// We can solve this problem by ignoring fsnotify.Chmod event.
if event.Op&fsnotify.Write != fsnotify.Write {
continue
}
c, err := w.f.Read()
if err != nil {
return nil, err
}
return c, nil
case err := <-w.fw.Errors:
return nil, err
case <-w.exit:
return nil, source.ErrWatcherStopped
}
}
}
func (w *watcher) Stop() error {
return w.fw.Close()
}

80
config/source/file/watcher_linux.go
View File

@@ -1,80 +0,0 @@
//+build linux
package file
import (
"os"
"reflect"
"github.com/fsnotify/fsnotify"
"github.com/micro/go-micro/v3/config/source"
)
type watcher struct {
f *file
fw *fsnotify.Watcher
exit chan bool
}
func newWatcher(f *file) (source.Watcher, error) {
fw, err := fsnotify.NewWatcher()
if err != nil {
return nil, err
}
fw.Add(f.path)
return &watcher{
f: f,
fw: fw,
exit: make(chan bool),
}, nil
}
func (w *watcher) Next() (*source.ChangeSet, error) {
// is it closed?
select {
case <-w.exit:
return nil, source.ErrWatcherStopped
default:
}
for {
// try get the event
select {
case event, _ := <-w.fw.Events:
if event.Op == fsnotify.Rename {
// check existence of file, and add watch again
_, err := os.Stat(event.Name)
if err == nil || os.IsExist(err) {
w.fw.Add(event.Name)
}
}
c, err := w.f.Read()
if err != nil {
return nil, err
}
// ARCH: Linux centos-7.shared 3.10.0-693.5.2.el7.x86_64
// Sometimes, ioutil.WriteFile triggers multiple fsnotify.Write events, which may be a bug.
// Detect if the file has changed
if reflect.DeepEqual(c.Data, w.f.data) {
continue
}
w.f.data = c.Data
return c, nil
case err := <-w.fw.Errors:
return nil, err
case <-w.exit:
return nil, source.ErrWatcherStopped
}
}
}
func (w *watcher) Stop() error {
return w.fw.Close()
}

47
config/source/flag/README.md
View File

@@ -1,47 +0,0 @@
# Flag Source
The flag source reads config from flags
## Format
We expect the use of the `flag` package. Upper case flags will be lower cased. Dashes will be used as delimiters.
### Example
```
dbAddress := flag.String("database_address", "127.0.0.1", "the db address")
dbPort := flag.Int("database_port", 3306, "the db port)
```
Becomes
```json
{
"database": {
"address": "127.0.0.1",
"port": 3306
}
}
```
## New Source
```go
flagSource := flag.NewSource(
// optionally enable reading of unset flags and their default
// values into config, defaults to false
IncludeUnset(true)
)
```
## Load Source
Load the source into config
```go
// Create new config
conf := config.NewConfig()
// Load flag source
conf.Load(flagSource)
```

101
config/source/flag/flag.go
View File

@@ -1,101 +0,0 @@
package flag
import (
"errors"
"flag"
"github.com/imdario/mergo"
"github.com/micro/go-micro/v3/config/source"
"strings"
"time"
)
type flagsrc struct {
opts source.Options
}
func (fs *flagsrc) Read() (*source.ChangeSet, error) {
if !flag.Parsed() {
return nil, errors.New("flags not parsed")
}
var changes map[string]interface{}
visitFn := func(f *flag.Flag) {
n := strings.ToLower(f.Name)
keys := strings.FieldsFunc(n, split)
reverse(keys)
tmp := make(map[string]interface{})
for i, k := range keys {
if i == 0 {
tmp[k] = f.Value
continue
}
tmp = map[string]interface{}{k: tmp}
}
mergo.Map(&changes, tmp) // need to sort error handling
return
}
unset, ok := fs.opts.Context.Value(includeUnsetKey{}).(bool)
if ok && unset {
flag.VisitAll(visitFn)
} else {
flag.Visit(visitFn)
}
b, err := fs.opts.Encoder.Encode(changes)
if err != nil {
return nil, err
}
cs := &source.ChangeSet{
Format: fs.opts.Encoder.String(),
Data: b,
Timestamp: time.Now(),
Source: fs.String(),
}
cs.Checksum = cs.Sum()
return cs, nil
}
func split(r rune) bool {
return r == '-' || r == '_'
}
func reverse(ss []string) {
for i := len(ss)/2 - 1; i >= 0; i-- {
opp := len(ss) - 1 - i
ss[i], ss[opp] = ss[opp], ss[i]
}
}
func (fs *flagsrc) Watch() (source.Watcher, error) {
return source.NewNoopWatcher()
}
func (fs *flagsrc) Write(cs *source.ChangeSet) error {
return nil
}
func (fs *flagsrc) String() string {
return "flag"
}
// NewSource returns a config source for integrating parsed flags.
// Hyphens are delimiters for nesting, and all keys are lowercased.
//
// Example:
// dbhost := flag.String("database-host", "localhost", "the db host name")
//
// {
// "database": {
// "host": "localhost"
// }
// }
func NewSource(opts ...source.Option) source.Source {
return &flagsrc{opts: source.NewOptions(opts...)}
}

68
config/source/flag/flag_test.go
View File

@@ -1,68 +0,0 @@
package flag
import (
"encoding/json"
"flag"
"testing"
)
var (
dbuser = flag.String("database-user", "default", "db user")
dbhost = flag.String("database-host", "", "db host")
dbpw = flag.String("database-password", "", "db pw")
)
func initTestFlags() {
flag.Set("database-host", "localhost")
flag.Set("database-password", "some-password")
flag.Parse()
}
func TestFlagsrc_Read(t *testing.T) {
initTestFlags()
source := NewSource()
c, err := source.Read()
if err != nil {
t.Error(err)
}
var actual map[string]interface{}
if err := json.Unmarshal(c.Data, &actual); err != nil {
t.Error(err)
}
actualDB := actual["database"].(map[string]interface{})
if actualDB["host"] != *dbhost {
t.Errorf("expected %v got %v", *dbhost, actualDB["host"])
}
if actualDB["password"] != *dbpw {
t.Errorf("expected %v got %v", *dbpw, actualDB["password"])
}
// unset flags should not be loaded
if actualDB["user"] != nil {
t.Errorf("expected %v got %v", nil, actualDB["user"])
}
}
func TestFlagsrc_ReadAll(t *testing.T) {
initTestFlags()
source := NewSource(IncludeUnset(true))
c, err := source.Read()
if err != nil {
t.Error(err)
}
var actual map[string]interface{}
if err := json.Unmarshal(c.Data, &actual); err != nil {
t.Error(err)
}
actualDB := actual["database"].(map[string]interface{})
// unset flag defaults should be loaded
if actualDB["user"] != *dbuser {
t.Errorf("expected %v got %v", *dbuser, actualDB["user"])
}
}

20
config/source/flag/options.go
View File

@@ -1,20 +0,0 @@
package flag
import (
"context"
"github.com/micro/go-micro/v3/config/source"
)
type includeUnsetKey struct{}
// IncludeUnset toggles the loading of unset flags and their respective default values.
// Default behavior is to ignore any unset flags.
func IncludeUnset(b bool) source.Option {
return func(o *source.Options) {
if o.Context == nil {
o.Context = context.Background()
}
o.Context = context.WithValue(o.Context, includeUnsetKey{}, true)
}
}

44
config/source/memory/README.md
View File

@@ -1,44 +0,0 @@
# Memory Source
The memory source provides in-memory data as a source
## Memory Format
The expected data format is json
```json
data := []byte(`{
"hosts": {
"database": {
"address": "10.0.0.1",
"port": 3306
},
"cache": {
"address": "10.0.0.2",
"port": 6379
}
}
}`)
```
## New Source
Specify source with data
```go
memorySource := memory.NewSource(
memory.WithJSON(data),
)
```
## Load Source
Load the source into config
```go
// Create new config
conf := config.NewConfig()
// Load memory source
conf.Load(memorySource)
```

99
config/source/memory/memory.go
View File

@@ -1,99 +0,0 @@
// Package memory is a memory source
package memory
import (
"sync"
"time"
"github.com/google/uuid"
"github.com/micro/go-micro/v3/config/source"
)
type memory struct {
sync.RWMutex
ChangeSet *source.ChangeSet
Watchers map[string]*watcher
}
func (s *memory) Read() (*source.ChangeSet, error) {
s.RLock()
cs := &source.ChangeSet{
Format: s.ChangeSet.Format,
Timestamp: s.ChangeSet.Timestamp,
Data: s.ChangeSet.Data,
Checksum: s.ChangeSet.Checksum,
Source: s.ChangeSet.Source,
}
s.RUnlock()
return cs, nil
}
func (s *memory) Watch() (source.Watcher, error) {
w := &watcher{
Id: uuid.New().String(),
Updates: make(chan *source.ChangeSet, 100),
Source: s,
}
s.Lock()
s.Watchers[w.Id] = w
s.Unlock()
return w, nil
}
func (m *memory) Write(cs *source.ChangeSet) error {
m.Update(cs)
return nil
}
// Update allows manual updates of the config data.
func (s *memory) Update(c *source.ChangeSet) {
// don't process nil
if c == nil {
return
}
// hash the file
s.Lock()
// update changeset
s.ChangeSet = &source.ChangeSet{
Data: c.Data,
Format: c.Format,
Source: "memory",
Timestamp: time.Now(),
}
s.ChangeSet.Checksum = s.ChangeSet.Sum()
// update watchers
for _, w := range s.Watchers {
select {
case w.Updates <- s.ChangeSet:
default:
}
}
s.Unlock()
}
func (s *memory) String() string {
return "memory"
}
func NewSource(opts ...source.Option) source.Source {
var options source.Options
for _, o := range opts {
o(&options)
}
s := &memory{
Watchers: make(map[string]*watcher),
}
if options.Context != nil {
c, ok := options.Context.Value(changeSetKey{}).(*source.ChangeSet)
if ok {
s.Update(c)
}
}
return s
}

41
config/source/memory/options.go
View File

@@ -1,41 +0,0 @@
package memory
import (
"context"
"github.com/micro/go-micro/v3/config/source"
)
type changeSetKey struct{}
func withData(d []byte, f string) source.Option {
return func(o *source.Options) {
if o.Context == nil {
o.Context = context.Background()
}
o.Context = context.WithValue(o.Context, changeSetKey{}, &source.ChangeSet{
Data: d,
Format: f,
})
}
}
// WithChangeSet allows a changeset to be set
func WithChangeSet(cs *source.ChangeSet) source.Option {
return func(o *source.Options) {
if o.Context == nil {
o.Context = context.Background()
}
o.Context = context.WithValue(o.Context, changeSetKey{}, cs)
}
}
// WithJSON allows the source data to be set to json
func WithJSON(d []byte) source.Option {
return withData(d, "json")
}
// WithYAML allows the source data to be set to yaml
func WithYAML(d []byte) source.Option {
return withData(d, "yaml")
}

23
config/source/memory/watcher.go
View File

@@ -1,23 +0,0 @@
package memory
import (
"github.com/micro/go-micro/v3/config/source"
)
type watcher struct {
Id string
Updates chan *source.ChangeSet
Source *memory
}
func (w *watcher) Next() (*source.ChangeSet, error) {
cs := <-w.Updates
return cs, nil
}
func (w *watcher) Stop() error {
w.Source.Lock()
delete(w.Source.Watchers, w.Id)
w.Source.Unlock()
return nil
}

25
config/source/noop.go
View File

@@ -1,25 +0,0 @@
package source
import (
"errors"
)
type noopWatcher struct {
exit chan struct{}
}
func (w *noopWatcher) Next() (*ChangeSet, error) {
<-w.exit
return nil, errors.New("noopWatcher stopped")
}
func (w *noopWatcher) Stop() error {
close(w.exit)
return nil
}
// NewNoopWatcher returns a watcher that blocks on Next() until Stop() is called.
func NewNoopWatcher() (Watcher, error) {
return &noopWatcher{exit: make(chan struct{})}, nil
}

49
config/source/options.go
View File

@@ -1,49 +0,0 @@
package source
import (
"context"
"github.com/micro/go-micro/v3/client"
"github.com/micro/go-micro/v3/config/encoder"
"github.com/micro/go-micro/v3/config/encoder/json"
)
type Options struct {
// Encoder
Encoder encoder.Encoder
// for alternative data
Context context.Context
// Client to use for RPC
Client client.Client
}
type Option func(o *Options)
func NewOptions(opts ...Option) Options {
options := Options{
Encoder: json.NewEncoder(),
Context: context.Background(),
}
for _, o := range opts {
o(&options)
}
return options
}
// WithEncoder sets the source encoder
func WithEncoder(e encoder.Encoder) Option {
return func(o *Options) {
o.Encoder = e
}
}
// WithClient sets the source client
func WithClient(c client.Client) Option {
return func(o *Options) {
o.Client = c
}
}

35
config/source/source.go
View File

@@ -1,35 +0,0 @@
// Package source is the interface for sources
package source
import (
"errors"
"time"
)
var (
// ErrWatcherStopped is returned when source watcher has been stopped
ErrWatcherStopped = errors.New("watcher stopped")
)
// Source is the source from which config is loaded
type Source interface {
Read() (*ChangeSet, error)
Write(*ChangeSet) error
Watch() (Watcher, error)
String() string
}
// ChangeSet represents a set of changes from a source
type ChangeSet struct {
Data []byte
Checksum string
Format string
Source string
Timestamp time.Time
}
// Watcher watches a source for changes
type Watcher interface {
Next() (*ChangeSet, error)
Stop() error
}

66
config/store/store.go Normal file
View File

@@ -0,0 +1,66 @@
package storeconfig
import (
"github.com/micro/go-micro/v3/config"
"github.com/micro/go-micro/v3/store"
)
// NewConfig returns new config
func NewConfig(store store.Store, key string) (config.Config, error) {
return newConfig(store)
}
type conf struct {
key string
store store.Store
}
func newConfig(store store.Store) (*conf, error) {
return &conf{
store: store,
}, nil
}
func mergeOptions(old config.Options, nu ...config.Option) config.Options {
n := config.Options{
Secret: old.Secret,
}
for _, opt := range nu {
opt(&n)
}
return n
}
func (c *conf) Get(path string, options ...config.Option) config.Value {
rec, err := c.store.Read(c.key)
dat := []byte("{}")
if err == nil && len(rec) > 0 {
dat = rec[0].Value
}
values := config.NewJSONValues(dat)
return values.Get(path)
}
func (c *conf) Set(path string, val interface{}, options ...config.Option) {
rec, err := c.store.Read(c.key)
dat := []byte("{}")
if err == nil && len(rec) > 0 {
dat = rec[0].Value
}
values := config.NewJSONValues(dat)
values.Set(path, val)
c.store.Write(&store.Record{
Key: c.key,
Value: values.Bytes(),
})
}
func (c *conf) Delete(path string, options ...config.Option) {
rec, err := c.store.Read(c.key)
dat := []byte("{}")
if err != nil || len(rec) == 0 {
return
}
values := config.NewJSONValues(dat)
values.Delete(path)
}

230
config/value.go
View File

@@ -1,49 +1,215 @@
package config
import (
"encoding/json"
"fmt"
"strconv"
"strings"
"time"
"github.com/micro/go-micro/v3/config/reader"
simple "github.com/bitly/go-simplejson"
)
type value struct{}
func newValue() reader.Value {
return new(value)
type JSONValues struct {
values []byte
sj *simple.Json
}
func (v *value) Bool(def bool) bool {
type JSONValue struct {
*simple.Json
}
func NewJSONValues(data []byte) *JSONValues {
sj := simple.New()
if err := sj.UnmarshalJSON(data); err != nil {
sj.SetPath(nil, string(data))
}
return &JSONValues{data, sj}
}
func NewJSONValue(data []byte) *JSONValue {
sj := simple.New()
if err := sj.UnmarshalJSON(data); err != nil {
sj.SetPath(nil, string(data))
}
return &JSONValue{sj}
}
func (j *JSONValues) Get(path string, options ...Option) Value {
paths := strings.Split(path, ".")
return &JSONValue{j.sj.GetPath(paths...)}
}
func (j *JSONValues) Delete(path string, options ...Option) {
paths := strings.Split(path, ".")
// delete the tree?
if len(paths) == 0 {
j.sj = simple.New()
return
}
if len(paths) == 1 {
j.sj.Del(paths[0])
return
}
vals := j.sj.GetPath(paths[:len(paths)-1]...)
vals.Del(paths[len(paths)-1])
j.sj.SetPath(paths[:len(paths)-1], vals.Interface())
return
}
func (j *JSONValues) Set(path string, val interface{}, options ...Option) {
paths := strings.Split(path, ".")
j.sj.SetPath(paths, val)
}
func (j *JSONValues) Bytes() []byte {
b, _ := j.sj.MarshalJSON()
return b
}
func (j *JSONValues) Map() map[string]interface{} {
m, _ := j.sj.Map()
return m
}
func (j *JSONValues) Scan(v interface{}) error {
b, err := j.sj.MarshalJSON()
if err != nil {
return err
}
return json.Unmarshal(b, v)
}
func (j *JSONValues) String() string {
return "json"
}
func (j *JSONValue) Bool(def bool) bool {
b, err := j.Json.Bool()
if err == nil {
return b
}
str, ok := j.Interface().(string)
if !ok {
return def
}
b, err = strconv.ParseBool(str)
if err != nil {
return def
}
return b
}
func (j *JSONValue) Int(def int) int {
i, err := j.Json.Int()
if err == nil {
return i
}
str, ok := j.Interface().(string)
if !ok {
return def
}
i, err = strconv.Atoi(str)
if err != nil {
return def
}
return i
}
func (j *JSONValue) String(def string) string {
return j.Json.MustString(def)
}
func (j *JSONValue) Float64(def float64) float64 {
f, err := j.Json.Float64()
if err == nil {
return f
}
str, ok := j.Interface().(string)
if !ok {
return def
}
f, err = strconv.ParseFloat(str, 64)
if err != nil {
return def
}
return f
}
func (j *JSONValue) Duration(def time.Duration) time.Duration {
v, err := j.Json.String()
if err != nil {
return def
}
value, err := time.ParseDuration(v)
if err != nil {
return def
}
return value
}
func (j *JSONValue) StringSlice(def []string) []string {
v, err := j.Json.String()
if err == nil {
sl := strings.Split(v, ",")
if len(sl) > 1 {
return sl
}
}
return j.Json.MustStringArray(def)
}
func (j *JSONValue) Exists() bool {
return false
}
func (v *value) Int(def int) int {
return 0
func (j *JSONValue) StringMap(def map[string]string) map[string]string {
m, err := j.Json.Map()
if err != nil {
return def
}
res := map[string]string{}
for k, v := range m {
res[k] = fmt.Sprintf("%v", v)
}
return res
}
func (v *value) String(def string) string {
return ""
func (j *JSONValue) Scan(v interface{}) error {
b, err := j.Json.MarshalJSON()
if err != nil {
return err
}
return json.Unmarshal(b, v)
}
func (v *value) Float64(def float64) float64 {
return 0.0
}
func (v *value) Duration(def time.Duration) time.Duration {
return time.Duration(0)
}
func (v *value) StringSlice(def []string) []string {
return nil
}
func (v *value) StringMap(def map[string]string) map[string]string {
return map[string]string{}
}
func (v *value) Scan(val interface{}) error {
return nil
}
func (v *value) Bytes() []byte {
return nil
func (j *JSONValue) Bytes() []byte {
b, err := j.Json.Bytes()
if err != nil {
// try return marshalled
b, err = j.Json.MarshalJSON()
if err != nil {
return []byte{}
}
return b
}
return b
}