Compare commits

...

6 Commits

Author SHA1 Message Date
ae787511f4 fixup load options using
Some checks failed
build / test (push) Has been cancelled
build / lint (push) Has been cancelled
codeql / analyze (go) (push) Failing after 2m30s
Signed-off-by: Vasiliy Tolstov <v.tolstov@unistack.org>
2024-04-17 16:56:21 +03:00
8368768957 add text transform support
Some checks are pending
build / test (push) Waiting to run
build / lint (push) Waiting to run
codeql / analyze (go) (push) Waiting to run
Signed-off-by: Vasiliy Tolstov <v.tolstov@unistack.org>
2024-04-17 16:49:21 +03:00
2fa6be8f5a update deps
Some checks failed
build / test (push) Failing after 1m8s
codeql / analyze (go) (push) Failing after 1m53s
build / lint (push) Successful in 9m37s
Signed-off-by: Vasiliy Tolstov <v.tolstov@unistack.org>
2024-03-06 17:34:29 +03:00
d77095fb4c fixup conditions
Some checks failed
build / test (push) Has been cancelled
build / lint (push) Has been cancelled
codeql / analyze (go) (push) Has been cancelled
Signed-off-by: Vasiliy Tolstov <v.tolstov@unistack.org>
2024-01-15 01:59:42 +03:00
4dc1cb3e1c Merge pull request 'update for latest micro changes' (#96) from microv4 into master
Some checks failed
build / test (push) Failing after 1m28s
build / lint (push) Failing after 2m32s
codeql / analyze (go) (push) Failing after 3m7s
Reviewed-on: #96
2023-08-14 23:50:39 +03:00
38851a57bf update for latest micro changes
Some checks failed
dependabot-automerge / automerge (pull_request) Has been skipped
prbuild / test (pull_request) Failing after 1m29s
prbuild / lint (pull_request) Failing after 2m37s
autoapprove / autoapprove (pull_request) Failing after 1m25s
automerge / automerge (pull_request) Failing after 5s
codeql / analyze (go) (pull_request) Failing after 3m8s
Signed-off-by: Vasiliy Tolstov <v.tolstov@unistack.org>
2023-08-14 23:50:12 +03:00
5 changed files with 277 additions and 34 deletions

169
file.go
View File

@ -1,16 +1,18 @@
package file // import "go.unistack.org/micro-config-file/v4"
package file
import (
"context"
"fmt"
"io"
"io/ioutil"
"os"
"regexp"
"github.com/imdario/mergo"
"dario.cat/mergo"
"go.unistack.org/micro/v4/codec"
"go.unistack.org/micro/v4/config"
"go.unistack.org/micro/v4/options"
rutil "go.unistack.org/micro/v4/util/reflect"
"golang.org/x/text/transform"
)
var DefaultStructTag = "file"
@ -18,29 +20,45 @@ var DefaultStructTag = "file"
type fileConfig struct {
opts config.Options
path string
reader io.Reader
transformer transform.Transformer
}
func (c *fileConfig) Options() config.Options {
return c.opts
}
func (c *fileConfig) Init(opts ...config.Option) error {
if err := config.DefaultBeforeInit(c.opts.Context, c); err != nil && !c.opts.AllowFail {
func (c *fileConfig) Init(opts ...options.Option) error {
var err error
if err = config.DefaultBeforeInit(c.opts.Context, c); err != nil && !c.opts.AllowFail {
return err
}
for _, o := range opts {
o(&c.opts)
if err = o(&c.opts); err != nil {
return err
}
}
if c.opts.Context != nil {
if v, ok := c.opts.Context.Value(pathKey{}).(string); ok {
c.path = v
}
if v, ok := c.opts.Context.Value(transformerKey{}).(transform.Transformer); ok {
c.transformer = v
}
if v, ok := c.opts.Context.Value(readerKey{}).(io.Reader); ok {
c.reader = v
}
}
if c.path == "" {
err := fmt.Errorf("file path not exists: %v", c.path)
if c.opts.Codec == nil {
return fmt.Errorf("Codec must be specified")
}
if c.path == "" && c.reader == nil {
err := fmt.Errorf("Path or Reader must be specified")
if !c.opts.AllowFail {
return err
}
@ -53,23 +71,50 @@ func (c *fileConfig) Init(opts ...config.Option) error {
return nil
}
func (c *fileConfig) Load(ctx context.Context, opts ...config.LoadOption) error {
func (c *fileConfig) Load(ctx context.Context, opts ...options.Option) error {
if c.opts.SkipLoad != nil && c.opts.SkipLoad(ctx, c) {
return nil
}
if err := config.DefaultBeforeLoad(ctx, c); err != nil && !c.opts.AllowFail {
return err
}
path := c.path
transformer := c.transformer
reader := c.reader
options := config.NewLoadOptions(opts...)
if options.Context != nil {
if v, ok := options.Context.Value(pathKey{}).(string); ok && v != "" {
path = v
}
if v, ok := c.opts.Context.Value(transformerKey{}).(transform.Transformer); ok {
transformer = v
}
if v, ok := c.opts.Context.Value(readerKey{}).(io.Reader); ok {
reader = v
}
}
var fp io.Reader
var err error
if c.path != "" {
fp, err = os.OpenFile(path, os.O_RDONLY, os.FileMode(0o400))
} else if c.reader != nil {
fp = reader
} else {
err = fmt.Errorf("Path or Reader must be specified")
}
fp, err := os.OpenFile(path, os.O_RDONLY, os.FileMode(0400))
if err != nil {
if !c.opts.AllowFail {
if c.path != "" {
return fmt.Errorf("file load path %s error: %w", path, err)
} else {
return fmt.Errorf("file load error: %w", err)
}
}
if err = config.DefaultAfterLoad(ctx, c); err != nil && !c.opts.AllowFail {
return err
@ -78,9 +123,18 @@ func (c *fileConfig) Load(ctx context.Context, opts ...config.LoadOption) error
return nil
}
defer fp.Close()
if fpc, ok := fp.(io.Closer); ok {
defer fpc.Close()
}
buf, err := ioutil.ReadAll(io.LimitReader(fp, int64(codec.DefaultMaxMsgSize)))
var r io.Reader
if transformer != nil {
r = transform.NewReader(fp, c.transformer)
} else {
r = fp
}
buf, err := io.ReadAll(io.LimitReader(r, int64(codec.DefaultMaxMsgSize)))
if err != nil {
if !c.opts.AllowFail {
return err
@ -124,7 +178,11 @@ func (c *fileConfig) Load(ctx context.Context, opts ...config.LoadOption) error
return nil
}
func (c *fileConfig) Save(ctx context.Context, opts ...config.SaveOption) error {
func (c *fileConfig) Save(ctx context.Context, opts ...options.Option) error {
if c.opts.SkipSave != nil && c.opts.SkipSave(ctx, c) {
return nil
}
if err := config.DefaultBeforeSave(ctx, c); err != nil && !c.opts.AllowFail {
return err
}
@ -154,7 +212,7 @@ func (c *fileConfig) Save(ctx context.Context, opts ...config.SaveOption) error
return nil
}
fp, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE, os.FileMode(0600))
fp, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE, os.FileMode(0o600))
if err != nil {
if !c.opts.AllowFail {
return err
@ -190,7 +248,7 @@ func (c *fileConfig) Name() string {
return c.opts.Name
}
func (c *fileConfig) Watch(ctx context.Context, opts ...config.WatchOption) (config.Watcher, error) {
func (c *fileConfig) Watch(ctx context.Context, opts ...options.Option) (config.Watcher, error) {
path := c.path
options := config.NewWatchOptions(opts...)
if options.Context != nil {
@ -213,10 +271,89 @@ func (c *fileConfig) Watch(ctx context.Context, opts ...config.WatchOption) (con
return w, nil
}
func NewConfig(opts ...config.Option) config.Config {
func NewConfig(opts ...options.Option) config.Config {
options := config.NewOptions(opts...)
if len(options.StructTag) == 0 {
options.StructTag = DefaultStructTag
}
return &fileConfig{opts: options}
}
type EnvTransformer struct {
maxMatchSize int
Regexp *regexp.Regexp
TransformerFunc TransformerFunc
overflow []byte
}
var _ transform.Transformer = (*EnvTransformer)(nil)
// Transform implements golang.org/x/text/transform#Transformer
func (t *EnvTransformer) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
t.maxMatchSize = 1024
var n int
// copy any overflow from the last call
if len(t.overflow) > 0 {
n, err = fullcopy(dst, t.overflow)
nDst += n
if err != nil {
t.overflow = t.overflow[n:]
return
}
t.overflow = nil
}
for _, index := range t.Regexp.FindAllSubmatchIndex(src, -1) {
// copy everything up to the match
n, err = fullcopy(dst[nDst:], src[nSrc:index[0]])
nSrc += n
nDst += n
if err != nil {
return
}
// skip the match if it ends at the end the src buffer.
// it could potentially match more
if index[1] == len(src) && !atEOF {
break
}
// copy the replacement
rep := t.TransformerFunc(src, index)
n, err = fullcopy(dst[nDst:], rep)
nDst += n
nSrc = index[1]
if err != nil {
t.overflow = rep[n:]
return
}
}
// if we're at the end, tack on any remaining bytes
if atEOF {
n, err = fullcopy(dst[nDst:], src[nSrc:])
nDst += n
nSrc += n
return
}
// skip any bytes which exceede the max match size
if skip := len(src[nSrc:]) - t.maxMatchSize; skip > 0 {
n, err = fullcopy(dst[nDst:], src[nSrc:nSrc+skip])
nSrc += n
nDst += n
if err != nil {
return
}
}
err = transform.ErrShortSrc
return
}
// Reset resets the state and allows a Transformer to be reused.
func (t *EnvTransformer) Reset() {
t.overflow = nil
}
func fullcopy(dst, src []byte) (n int, err error) {
n = copy(dst, src)
if n < len(src) {
err = transform.ErrShortDst
}
return
}

73
file_test.go Normal file
View File

@ -0,0 +1,73 @@
package file
import (
"bytes"
"context"
"encoding/json"
"io"
"os"
"testing"
"go.unistack.org/micro/v4/codec"
"go.unistack.org/micro/v4/config"
"go.unistack.org/micro/v4/options"
)
type jsoncodec struct{}
func (*jsoncodec) Marshal(v interface{}, opts ...codec.Option) ([]byte, error) {
return json.Marshal(v)
}
func (*jsoncodec) Unmarshal(buf []byte, v interface{}, opts ...codec.Option) error {
return json.Unmarshal(buf, v)
}
func (*jsoncodec) ReadBody(r io.Reader, v interface{}) error {
return nil
}
func (*jsoncodec) ReadHeader(r io.Reader, m *codec.Message, t codec.MessageType) error {
return nil
}
func (*jsoncodec) String() string {
return "json"
}
func (*jsoncodec) Write(w io.Writer, m *codec.Message, v interface{}) error {
return nil
}
func TestLoadReplace(t *testing.T) {
type Config struct {
Key string
Pass string
}
os.Setenv("PLACEHOLDER", "test")
cfg := &Config{}
ctx := context.TODO()
buf := bytes.NewReader([]byte(`{"key":"val","pass":"${PLACEHOLDER}"}`))
tr, err := NewEnvTransformer(`(?s)\$\{.*?\}`, 2, 1)
if err != nil {
t.Fatal(err)
}
c := NewConfig(options.Codec(
&jsoncodec{}),
config.Struct(cfg),
Reader(buf),
Transformer(tr),
)
if err := c.Init(); err != nil {
t.Fatal(err)
}
if err := c.Load(ctx); err != nil {
t.Fatal(err)
}
if cfg.Pass != "test" {
t.Fatalf("not works %#+v\n", cfg)
}
}

6
go.mod
View File

@ -3,6 +3,8 @@ module go.unistack.org/micro-config-file/v4
go 1.20
require (
github.com/imdario/mergo v0.3.14
go.unistack.org/micro/v4 v4.0.1
dario.cat/mergo v1.0.0
go.unistack.org/micro/v4 v4.0.17
)
require github.com/google/uuid v1.6.0 // indirect

10
go.sum
View File

@ -1,7 +1,9 @@
github.com/imdario/mergo v0.3.14 h1:fOqeC1+nCuuk6PKQdg9YmosXX7Y7mHX6R/0ZldI9iHo=
github.com/imdario/mergo v0.3.14/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY=
go.unistack.org/micro/v4 v4.0.1 h1:xo1IxbVfgh8i0eY0VeYa3cbb13u5n/Mxnp3FOgWD4Jo=
go.unistack.org/micro/v4 v4.0.1/go.mod h1:p/J5UcSJjfHsWGT31uKoghQ5rUQZzQJBAFy+Z4+ZVMs=
dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk=
dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
go.unistack.org/micro/v4 v4.0.17 h1:mF7uM+J4ILdG+1fcwzKYCwDlxhdbF/e1WnGzKKLnIXc=
go.unistack.org/micro/v4 v4.0.17/go.mod h1:ZDgU9931vm2l7X6RN/6UuwRIVp24GRdmQ7dKmegArk4=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

View File

@ -1,23 +1,52 @@
package file
import (
"go.unistack.org/micro/v4/config"
"io"
"os"
"regexp"
"go.unistack.org/micro/v4/options"
"golang.org/x/text/transform"
)
type pathKey struct{}
func Path(path string) config.Option {
return config.SetOption(pathKey{}, path)
func Path(path string) options.Option {
return options.ContextOption(pathKey{}, path)
}
func LoadPath(path string) config.LoadOption {
return config.SetLoadOption(pathKey{}, path)
type readerKey struct{}
func Reader(r io.Reader) options.Option {
return options.ContextOption(readerKey{}, r)
}
func SavePath(path string) config.SaveOption {
return config.SetSaveOption(pathKey{}, path)
type transformerKey struct{}
type TransformerFunc func(src []byte, index []int) []byte
func Transformer(t transform.Transformer) options.Option {
return options.ContextOption(transformerKey{}, t)
}
func WatchPath(path string) config.WatchOption {
return config.SetWatchOption(pathKey{}, path)
func NewEnvTransformer(rs string, trimLeft, trimRight int) (*EnvTransformer, error) {
re, err := regexp.Compile(rs)
if err != nil {
return nil, err
}
return &EnvTransformer{
Regexp: re,
TransformerFunc: func(src []byte, index []int) []byte {
var envKey string
if len(src) > index[1]-trimRight {
envKey = string(src[index[0]+trimLeft : index[1]-trimRight])
}
if envVal, ok := os.LookupEnv(envKey); ok {
return []byte(envVal)
}
return src[index[0]:index[1]]
},
}, nil
}