From 836876895708b96c3c12c06e19ce08e84b1d8634 Mon Sep 17 00:00:00 2001 From: Vasiliy Tolstov Date: Wed, 17 Apr 2024 14:57:30 +0300 Subject: [PATCH] add text transform support Signed-off-by: Vasiliy Tolstov --- file.go | 140 +++++++++++++++++++++++++++++++++++++++++++++++---- file_test.go | 73 +++++++++++++++++++++++++++ options.go | 41 +++++++++++++++ 3 files changed, 244 insertions(+), 10 deletions(-) create mode 100644 file_test.go diff --git a/file.go b/file.go index 3db148b..5a1712f 100644 --- a/file.go +++ b/file.go @@ -5,19 +5,23 @@ import ( "fmt" "io" "os" + "regexp" "dario.cat/mergo" "go.unistack.org/micro/v4/codec" "go.unistack.org/micro/v4/config" "go.unistack.org/micro/v4/options" rutil "go.unistack.org/micro/v4/util/reflect" + "golang.org/x/text/transform" ) var DefaultStructTag = "file" type fileConfig struct { - opts config.Options - path string + opts config.Options + path string + reader io.Reader + transformer transform.Transformer } func (c *fileConfig) Options() config.Options { @@ -25,22 +29,36 @@ func (c *fileConfig) Options() config.Options { } func (c *fileConfig) Init(opts ...options.Option) error { - if err := config.DefaultBeforeInit(c.opts.Context, c); err != nil && !c.opts.AllowFail { + var err error + + if err = config.DefaultBeforeInit(c.opts.Context, c); err != nil && !c.opts.AllowFail { return err } for _, o := range opts { - o(&c.opts) + if err = o(&c.opts); err != nil { + return err + } } if c.opts.Context != nil { if v, ok := c.opts.Context.Value(pathKey{}).(string); ok { c.path = v } + if v, ok := c.opts.Context.Value(transformerKey{}).(transform.Transformer); ok { + c.transformer = v + } + if v, ok := c.opts.Context.Value(readerKey{}).(io.Reader); ok { + c.reader = v + } } - if c.path == "" { - err := fmt.Errorf("file path not exists: %v", c.path) + if c.opts.Codec == nil { + return fmt.Errorf("Codec must be specified") + } + + if c.path == "" && c.reader == nil { + err := fmt.Errorf("Path or Reader must be specified") if !c.opts.AllowFail { return err } @@ -70,10 +88,24 @@ func (c *fileConfig) Load(ctx context.Context, opts ...options.Option) error { } } - fp, err := os.OpenFile(path, os.O_RDONLY, os.FileMode(0o400)) + var fp io.Reader + var err error + + if c.path != "" { + fp, err = os.OpenFile(path, os.O_RDONLY, os.FileMode(0o400)) + } else if c.reader != nil { + fp = c.reader + } else { + err = fmt.Errorf("Path or Reader must be specified") + } + if err != nil { if !c.opts.AllowFail { - return fmt.Errorf("file load path %s error: %w", path, err) + if c.path != "" { + return fmt.Errorf("file load path %s error: %w", path, err) + } else { + return fmt.Errorf("file load error: %w", err) + } } if err = config.DefaultAfterLoad(ctx, c); err != nil && !c.opts.AllowFail { return err @@ -82,9 +114,18 @@ func (c *fileConfig) Load(ctx context.Context, opts ...options.Option) error { return nil } - defer fp.Close() + if fpc, ok := fp.(io.Closer); ok { + defer fpc.Close() + } - buf, err := io.ReadAll(io.LimitReader(fp, int64(codec.DefaultMaxMsgSize))) + var r io.Reader + if c.transformer != nil { + r = transform.NewReader(fp, c.transformer) + } else { + r = fp + } + + buf, err := io.ReadAll(io.LimitReader(r, int64(codec.DefaultMaxMsgSize))) if err != nil { if !c.opts.AllowFail { return err @@ -228,3 +269,82 @@ func NewConfig(opts ...options.Option) config.Config { } return &fileConfig{opts: options} } + +type EnvTransformer struct { + maxMatchSize int + Regexp *regexp.Regexp + TransformerFunc TransformerFunc + overflow []byte +} + +var _ transform.Transformer = (*EnvTransformer)(nil) + +// Transform implements golang.org/x/text/transform#Transformer +func (t *EnvTransformer) Transform(dst, src []byte, atEOF bool) (nDst, nSrc int, err error) { + t.maxMatchSize = 1024 + var n int + // copy any overflow from the last call + if len(t.overflow) > 0 { + n, err = fullcopy(dst, t.overflow) + nDst += n + if err != nil { + t.overflow = t.overflow[n:] + return + } + t.overflow = nil + } + for _, index := range t.Regexp.FindAllSubmatchIndex(src, -1) { + // copy everything up to the match + n, err = fullcopy(dst[nDst:], src[nSrc:index[0]]) + nSrc += n + nDst += n + if err != nil { + return + } + // skip the match if it ends at the end the src buffer. + // it could potentially match more + if index[1] == len(src) && !atEOF { + break + } + // copy the replacement + rep := t.TransformerFunc(src, index) + n, err = fullcopy(dst[nDst:], rep) + nDst += n + nSrc = index[1] + if err != nil { + t.overflow = rep[n:] + return + } + } + // if we're at the end, tack on any remaining bytes + if atEOF { + n, err = fullcopy(dst[nDst:], src[nSrc:]) + nDst += n + nSrc += n + return + } + // skip any bytes which exceede the max match size + if skip := len(src[nSrc:]) - t.maxMatchSize; skip > 0 { + n, err = fullcopy(dst[nDst:], src[nSrc:nSrc+skip]) + nSrc += n + nDst += n + if err != nil { + return + } + } + err = transform.ErrShortSrc + return +} + +// Reset resets the state and allows a Transformer to be reused. +func (t *EnvTransformer) Reset() { + t.overflow = nil +} + +func fullcopy(dst, src []byte) (n int, err error) { + n = copy(dst, src) + if n < len(src) { + err = transform.ErrShortDst + } + return +} diff --git a/file_test.go b/file_test.go new file mode 100644 index 0000000..befb08b --- /dev/null +++ b/file_test.go @@ -0,0 +1,73 @@ +package file + +import ( + "bytes" + "context" + "encoding/json" + "io" + "os" + "testing" + + "go.unistack.org/micro/v4/codec" + "go.unistack.org/micro/v4/config" + "go.unistack.org/micro/v4/options" +) + +type jsoncodec struct{} + +func (*jsoncodec) Marshal(v interface{}, opts ...codec.Option) ([]byte, error) { + return json.Marshal(v) +} + +func (*jsoncodec) Unmarshal(buf []byte, v interface{}, opts ...codec.Option) error { + return json.Unmarshal(buf, v) +} + +func (*jsoncodec) ReadBody(r io.Reader, v interface{}) error { + return nil +} + +func (*jsoncodec) ReadHeader(r io.Reader, m *codec.Message, t codec.MessageType) error { + return nil +} + +func (*jsoncodec) String() string { + return "json" +} + +func (*jsoncodec) Write(w io.Writer, m *codec.Message, v interface{}) error { + return nil +} + +func TestLoadReplace(t *testing.T) { + type Config struct { + Key string + Pass string + } + os.Setenv("PLACEHOLDER", "test") + cfg := &Config{} + ctx := context.TODO() + buf := bytes.NewReader([]byte(`{"key":"val","pass":"${PLACEHOLDER}"}`)) + tr, err := NewEnvTransformer(`(?s)\$\{.*?\}`, 2, 1) + if err != nil { + t.Fatal(err) + } + c := NewConfig(options.Codec( + &jsoncodec{}), + config.Struct(cfg), + Reader(buf), + Transformer(tr), + ) + + if err := c.Init(); err != nil { + t.Fatal(err) + } + + if err := c.Load(ctx); err != nil { + t.Fatal(err) + } + + if cfg.Pass != "test" { + t.Fatalf("not works %#+v\n", cfg) + } +} diff --git a/options.go b/options.go index 79c6f73..6840624 100644 --- a/options.go +++ b/options.go @@ -1,7 +1,12 @@ package file import ( + "io" + "os" + "regexp" + "go.unistack.org/micro/v4/options" + "golang.org/x/text/transform" ) type pathKey struct{} @@ -9,3 +14,39 @@ type pathKey struct{} func Path(path string) options.Option { return options.ContextOption(pathKey{}, path) } + +type readerKey struct{} + +func Reader(r io.Reader) options.Option { + return options.ContextOption(readerKey{}, r) +} + +type transformerKey struct{} + +type TransformerFunc func(src []byte, index []int) []byte + +func Transformer(t transform.Transformer) options.Option { + return options.ContextOption(transformerKey{}, t) +} + +func NewEnvTransformer(rs string, trimLeft, trimRight int) (*EnvTransformer, error) { + re, err := regexp.Compile(rs) + if err != nil { + return nil, err + } + return &EnvTransformer{ + Regexp: re, + TransformerFunc: func(src []byte, index []int) []byte { + var envKey string + if len(src) > index[1]-trimRight { + envKey = string(src[index[0]+trimLeft : index[1]-trimRight]) + } + + if envVal, ok := os.LookupEnv(envKey); ok { + return []byte(envVal) + } + + return src[index[0]:index[1]] + }, + }, nil +}