Compare commits

...

4 Commits
v3.8.4 ... v3

Author SHA1 Message Date
89b41364c3 update to latest micro
Some checks failed
build / test (push) Failing after 9s
build / lint (push) Failing after 10s
codeql / analyze (go) (push) Failing after 10s
Signed-off-by: Vasiliy Tolstov <v.tolstov@unistack.org>
2024-09-17 13:09:10 +03:00
d7ffcd19f8 fixup load options using
Some checks failed
build / test (push) Has been cancelled
build / lint (push) Has been cancelled
codeql / analyze (go) (push) Has been cancelled
Signed-off-by: Vasiliy Tolstov <v.tolstov@unistack.org>
2024-04-17 16:55:13 +03:00
6b27204711 add text transform support
Some checks are pending
build / test (push) Waiting to run
build / lint (push) Waiting to run
codeql / analyze (go) (push) Waiting to run
Signed-off-by: Vasiliy Tolstov <v.tolstov@unistack.org>
2024-04-17 14:57:30 +03:00
4bee1a7041 update deps
Some checks failed
build / test (push) Failing after 1m6s
codeql / analyze (go) (push) Failing after 1m50s
build / lint (push) Successful in 9m21s
Signed-off-by: Vasiliy Tolstov <v.tolstov@unistack.org>
2024-03-06 17:32:33 +03:00
6 changed files with 249 additions and 31 deletions

152
file.go
View File

@ -4,20 +4,25 @@ import (
"context"
"fmt"
"io"
"io/ioutil"
"os"
"regexp"
"github.com/imdario/mergo"
"go.unistack.org/micro/v3/codec"
"dario.cat/mergo"
"go.unistack.org/micro/v3/config"
rutil "go.unistack.org/micro/v3/util/reflect"
"golang.org/x/text/transform"
)
var DefaultStructTag = "file"
var (
DefaultStructTag = "file"
MaxFileSize int64 = 1 * 1024 * 1024
)
type fileConfig struct {
opts config.Options
path string
opts config.Options
path string
reader io.Reader
transformer transform.Transformer
}
func (c *fileConfig) Options() config.Options {
@ -37,10 +42,20 @@ func (c *fileConfig) Init(opts ...config.Option) error {
if v, ok := c.opts.Context.Value(pathKey{}).(string); ok {
c.path = v
}
if v, ok := c.opts.Context.Value(transformerKey{}).(transform.Transformer); ok {
c.transformer = v
}
if v, ok := c.opts.Context.Value(readerKey{}).(io.Reader); ok {
c.reader = v
}
}
if c.path == "" {
err := fmt.Errorf("file path not exists: %v", c.path)
if c.opts.Codec == nil {
return fmt.Errorf("Codec must be specified")
}
if c.path == "" && c.reader == nil {
err := fmt.Errorf("Path or Reader must be specified")
if !c.opts.AllowFail {
return err
}
@ -63,17 +78,40 @@ func (c *fileConfig) Load(ctx context.Context, opts ...config.LoadOption) error
}
path := c.path
transformer := c.transformer
reader := c.reader
options := config.NewLoadOptions(opts...)
if options.Context != nil {
if v, ok := options.Context.Value(pathKey{}).(string); ok && v != "" {
path = v
}
if v, ok := c.opts.Context.Value(transformerKey{}).(transform.Transformer); ok {
transformer = v
}
if v, ok := c.opts.Context.Value(readerKey{}).(io.Reader); ok {
reader = v
}
}
var fp io.Reader
var err error
if c.path != "" {
fp, err = os.OpenFile(path, os.O_RDONLY, os.FileMode(0o400))
} else if c.reader != nil {
fp = reader
} else {
err = fmt.Errorf("Path or Reader must be specified")
}
fp, err := os.OpenFile(path, os.O_RDONLY, os.FileMode(0400))
if err != nil {
if !c.opts.AllowFail {
return fmt.Errorf("file load path %s error: %w", path, err)
if c.path != "" {
return fmt.Errorf("file load path %s error: %w", path, err)
} else {
return fmt.Errorf("file load error: %w", err)
}
}
if err = config.DefaultAfterLoad(ctx, c); err != nil && !c.opts.AllowFail {
return err
@ -82,9 +120,18 @@ func (c *fileConfig) Load(ctx context.Context, opts ...config.LoadOption) error
return nil
}
defer fp.Close()
if fpc, ok := fp.(io.Closer); ok {
defer fpc.Close()
}
buf, err := ioutil.ReadAll(io.LimitReader(fp, int64(codec.DefaultMaxMsgSize)))
var r io.Reader
if transformer != nil {
r = transform.NewReader(fp, c.transformer)
} else {
r = fp
}
buf, err := io.ReadAll(io.LimitReader(r, MaxFileSize))
if err != nil {
if !c.opts.AllowFail {
return err
@ -162,7 +209,7 @@ func (c *fileConfig) Save(ctx context.Context, opts ...config.SaveOption) error
return nil
}
fp, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE, os.FileMode(0600))
fp, err := os.OpenFile(path, os.O_WRONLY|os.O_CREATE, os.FileMode(0o600))
if err != nil {
if !c.opts.AllowFail {
return err
@ -228,3 +275,82 @@ func NewConfig(opts ...config.Option) config.Config {
}
return &fileConfig{opts: options}
}
type EnvTransformer struct {
maxMatchSize int
Regexp *regexp.Regexp
TransformerFunc TransformerFunc
overflow []byte
}
var _ transform.Transformer = (*EnvTransformer)(nil)
// Transform implements golang.org/x/text/transform#Transformer
func (t *EnvTransformer) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) {
t.maxMatchSize = 1024
var n int
// copy any overflow from the last call
if len(t.overflow) > 0 {
n, err = fullcopy(dst, t.overflow)
nDst += n
if err != nil {
t.overflow = t.overflow[n:]
return
}
t.overflow = nil
}
for _, index := range t.Regexp.FindAllSubmatchIndex(src, -1) {
// copy everything up to the match
n, err = fullcopy(dst[nDst:], src[nSrc:index[0]])
nSrc += n
nDst += n
if err != nil {
return
}
// skip the match if it ends at the end the src buffer.
// it could potentially match more
if index[1] == len(src) && !atEOF {
break
}
// copy the replacement
rep := t.TransformerFunc(src, index)
n, err = fullcopy(dst[nDst:], rep)
nDst += n
nSrc = index[1]
if err != nil {
t.overflow = rep[n:]
return
}
}
// if we're at the end, tack on any remaining bytes
if atEOF {
n, err = fullcopy(dst[nDst:], src[nSrc:])
nDst += n
nSrc += n
return
}
// skip any bytes which exceede the max match size
if skip := len(src[nSrc:]) - t.maxMatchSize; skip > 0 {
n, err = fullcopy(dst[nDst:], src[nSrc:nSrc+skip])
nSrc += n
nDst += n
if err != nil {
return
}
}
err = transform.ErrShortSrc
return
}
// Reset resets the state and allows a Transformer to be reused.
func (t *EnvTransformer) Reset() {
t.overflow = nil
}
func fullcopy(dst, src []byte) (n int, err error) {
n = copy(dst, src)
if n < len(src) {
err = transform.ErrShortDst
}
return
}

59
file_test.go Normal file
View File

@ -0,0 +1,59 @@
package file
import (
"bytes"
"context"
"encoding/json"
"os"
"testing"
"go.unistack.org/micro/v3/codec"
"go.unistack.org/micro/v3/config"
)
type jsoncodec struct{}
func (*jsoncodec) Marshal(v interface{}, opts ...codec.Option) ([]byte, error) {
return json.Marshal(v)
}
func (*jsoncodec) Unmarshal(buf []byte, v interface{}, opts ...codec.Option) error {
return json.Unmarshal(buf, v)
}
func (*jsoncodec) String() string {
return "json"
}
func TestLoadReplace(t *testing.T) {
type Config struct {
Key string
Pass string
}
os.Setenv("PLACEHOLDER", "test")
cfg := &Config{}
ctx := context.TODO()
buf := bytes.NewReader([]byte(`{"key":"val","pass":"${PLACEHOLDER}"}`))
tr, err := NewEnvTransformer(`(?s)\$\{.*?\}`, 2, 1)
if err != nil {
t.Fatal(err)
}
c := NewConfig(config.Codec(
&jsoncodec{}),
config.Struct(cfg),
Reader(buf),
Transformer(tr),
)
if err := c.Init(); err != nil {
t.Fatal(err)
}
if err := c.Load(ctx); err != nil {
t.Fatal(err)
}
if cfg.Pass != "test" {
t.Fatalf("not works %#+v\n", cfg)
}
}

6
go.mod
View File

@ -3,8 +3,8 @@ module go.unistack.org/micro-config-file/v3
go 1.18
require (
github.com/imdario/mergo v0.3.16
go.unistack.org/micro/v3 v3.10.36
dario.cat/mergo v1.0.0
go.unistack.org/micro/v3 v3.10.44
)
require github.com/google/uuid v1.5.0 // indirect
require github.com/google/uuid v1.6.0 // indirect

19
go.sum
View File

@ -1,16 +1,9 @@
github.com/google/uuid v1.5.0 h1:1p67kYwdtXjb0gL0BPiP1Av9wiZPo5A8z2cWkTZ+eyU=
github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/imdario/mergo v0.3.13 h1:lFzP57bqS/wsqKssCGmtLAb8A0wKjLGrve2q3PPVcBk=
github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg=
github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4=
github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY=
github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
github.com/silas/dag v0.0.0-20211117232152-9d50aa809f35/go.mod h1:7RTUFBdIRC9nZ7/3RyRNH1bdqIShrDejd1YbLwgPS+I=
go.unistack.org/micro/v3 v3.10.16 h1:2er/SKKYbV60M+UuJM4eYCF0MZYAIq/yNUrAbTfgq8Q=
go.unistack.org/micro/v3 v3.10.16/go.mod h1:uMAc0U/x7dmtICCrblGf0ZLgYegu3VwQAquu+OFCw1Q=
go.unistack.org/micro/v3 v3.10.36 h1:+XPhpguDC0HldSePp5TODdMJ06+IUy1AQkYN7ub84DA=
go.unistack.org/micro/v3 v3.10.36/go.mod h1:eUgtvbtiiz6te93m0ZdmoecbitWwjdBmmr84srmEIKA=
dario.cat/mergo v1.0.0 h1:AGCNq9Evsj31mOgNPcLyXc+4PNABt905YmuqPYYpBWk=
dario.cat/mergo v1.0.0/go.mod h1:uNxQE+84aUszobStD9th8a29P2fMDhsBdgRYvZOxGmk=
github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0=
github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
go.unistack.org/micro/v3 v3.10.44 h1:Vgyy9BrJOSdFvo29/klrgIBE/Nme9E8udPAljos34o0=
go.unistack.org/micro/v3 v3.10.44/go.mod h1:erMgt3Bl7vQQ0e9UpQyR5NlLiZ9pKeEJ9+1tfYFaqUg=
gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0=
gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=
gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA=
gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM=

View File

@ -1,7 +1,12 @@
package file
import (
"io"
"os"
"regexp"
"go.unistack.org/micro/v3/config"
"golang.org/x/text/transform"
)
type pathKey struct{}
@ -21,3 +26,39 @@ func SavePath(path string) config.SaveOption {
func WatchPath(path string) config.WatchOption {
return config.SetWatchOption(pathKey{}, path)
}
type readerKey struct{}
func Reader(r io.Reader) config.Option {
return config.SetOption(readerKey{}, r)
}
type transformerKey struct{}
type TransformerFunc func(src []byte, index []int) []byte
func Transformer(t transform.Transformer) config.Option {
return config.SetOption(transformerKey{}, t)
}
func NewEnvTransformer(rs string, trimLeft, trimRight int) (*EnvTransformer, error) {
re, err := regexp.Compile(rs)
if err != nil {
return nil, err
}
return &EnvTransformer{
Regexp: re,
TransformerFunc: func(src []byte, index []int) []byte {
var envKey string
if len(src) > index[1]-trimRight {
envKey = string(src[index[0]+trimLeft : index[1]-trimRight])
}
if envVal, ok := os.LookupEnv(envKey); ok {
return []byte(envVal)
}
return src[index[0]:index[1]]
},
}, nil
}

View File

@ -7,7 +7,6 @@ import (
"os"
"reflect"
"go.unistack.org/micro/v3/codec"
"go.unistack.org/micro/v3/config"
"go.unistack.org/micro/v3/util/jitter"
rutil "go.unistack.org/micro/v3/util/reflect"
@ -44,7 +43,7 @@ func (w *fileWatcher) run() {
return
}
var buf []byte
buf, err = ioutil.ReadAll(io.LimitReader(fp, int64(codec.DefaultMaxMsgSize)))
buf, err = ioutil.ReadAll(io.LimitReader(fp, MaxFileSize))
if err == nil {
err = w.opts.Codec.Unmarshal(buf, dst)
}