From d466a05b9de54b7fc98825f15f715d7ae70c3d30 Mon Sep 17 00:00:00 2001 From: Vasiliy Tolstov Date: Mon, 16 Sep 2024 23:44:26 +0300 Subject: [PATCH] update codec to allow reuse buffers Signed-off-by: Vasiliy Tolstov --- codec_test.go | 15 +---- go.mod | 7 +- go.sum | 19 ++---- options.go | 18 ++++++ proto.go | 175 +++++++++++++++++++++++++++++++++++--------------- 5 files changed, 151 insertions(+), 83 deletions(-) create mode 100644 options.go diff --git a/codec_test.go b/codec_test.go index 68e070f..81bcfdf 100644 --- a/codec_test.go +++ b/codec_test.go @@ -28,7 +28,7 @@ func TestFrameFlatten(t *testing.T) { Name: &codec.Frame{Data: []byte("test")}, } - buf, err := NewCodec().Marshal(s) + buf, err := NewCodec(codec.Flatten(true)).Marshal(s) if err != nil { t.Fatal(err) } @@ -36,16 +36,3 @@ func TestFrameFlatten(t *testing.T) { t.Fatalf("bytes not equal %s != %s", buf, `test`) } } - -func TestReadBody(t *testing.T) { - t.Skip("skip as no proto") - s := &struct { - Name string - }{} - c := NewCodec() - b := bytes.NewReader(nil) - err := c.ReadBody(b, s) - if err != nil { - t.Fatal(err) - } -} diff --git a/go.mod b/go.mod index 2c08176..a07375e 100644 --- a/go.mod +++ b/go.mod @@ -1,10 +1,11 @@ module go.unistack.org/micro-codec-proto/v3 -go 1.19 +go 1.22 require ( - go.unistack.org/micro/v3 v3.10.14 - google.golang.org/protobuf v1.28.1 + go.unistack.org/micro-proto/v3 v3.4.1 + go.unistack.org/micro/v3 v3.10.88 + google.golang.org/protobuf v1.34.2 ) require ( diff --git a/go.sum b/go.sum index f213052..219f567 100644 --- a/go.sum +++ b/go.sum @@ -1,18 +1,11 @@ -github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.7 h1:81/ik6ipDQS2aGcBfIN5dHDB36BwrStyeAQquSYCV4o= github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= -github.com/imdario/mergo v0.3.13/go.mod h1:4lJ1jqUDcsbIECGy0RUJAXNIhg+6ocWgb1ALK2O4oXg= -github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= -github.com/silas/dag v0.0.0-20211117232152-9d50aa809f35/go.mod h1:7RTUFBdIRC9nZ7/3RyRNH1bdqIShrDejd1YbLwgPS+I= -go.unistack.org/micro/v3 v3.10.14 h1:7fgLpwGlCN67twhwtngJDEQvrMkUBDSA5vzZqxIDqNE= -go.unistack.org/micro/v3 v3.10.14/go.mod h1:uMAc0U/x7dmtICCrblGf0ZLgYegu3VwQAquu+OFCw1Q= +go.unistack.org/micro-proto/v3 v3.4.1 h1:UTjLSRz2YZuaHk9iSlVqqsA50JQNAEK2ZFboGqtEa9Q= +go.unistack.org/micro-proto/v3 v3.4.1/go.mod h1:okx/cnOhzuCX0ggl/vToatbCupi0O44diiiLLsZ93Zo= +go.unistack.org/micro/v3 v3.10.88 h1:MxlzP+77Y6Kphb3lzHxROL4XfE/WdCQMQpnPv4D9Z8U= +go.unistack.org/micro/v3 v3.10.88/go.mod h1:erMgt3Bl7vQQ0e9UpQyR5NlLiZ9pKeEJ9+1tfYFaqUg= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= -google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= -google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= -google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= -gopkg.in/yaml.v3 v3.0.0/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= -gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +google.golang.org/protobuf v1.34.2 h1:6xV6lTsCfpGD21XK49h7MhtcApnLqkfYgPcdHftf6hg= +google.golang.org/protobuf v1.34.2/go.mod h1:qYOHts0dSfpeUzUFpOMr/WGzszTmLH+DiWniOlNbLDw= diff --git a/options.go b/options.go new file mode 100644 index 0000000..50dca91 --- /dev/null +++ b/options.go @@ -0,0 +1,18 @@ +package proto + +import ( + codec "go.unistack.org/micro/v3/codec" + "google.golang.org/protobuf/proto" +) + +type unmarshalOptionsKey struct{} + +func UnmarshalOptions(o proto.UnmarshalOptions) codec.Option { + return codec.SetOption(unmarshalOptionsKey{}, o) +} + +type marshalOptionsKey struct{} + +func MarshalOptions(o proto.MarshalOptions) codec.Option { + return codec.SetOption(marshalOptionsKey{}, o) +} diff --git a/proto.go b/proto.go index 5738fb2..9b99401 100644 --- a/proto.go +++ b/proto.go @@ -1,23 +1,35 @@ // Package proto provides a proto codec -package proto // import "go.unistack.org/micro-codec-proto/v3" +package proto import ( - "io" - pb "go.unistack.org/micro-proto/v3/codec" "go.unistack.org/micro/v3/codec" rutil "go.unistack.org/micro/v3/util/reflect" "google.golang.org/protobuf/proto" ) +var ( + DefaultMarshalOptions = proto.MarshalOptions{ + AllowPartial: false, + } + + DefaultUnmarshalOptions = proto.UnmarshalOptions{ + DiscardUnknown: false, + AllowPartial: false, + } +) + type protoCodec struct { opts codec.Options } -var _ codec.Codec = &protoCodec{} +type protoCodecV2 struct { + opts codec.Options +} -const ( - flattenTag = "flatten" +var ( + _ codec.Codec = (*protoCodec)(nil) + _ codec.CodecV2 = (*protoCodecV2)(nil) ) func (c *protoCodec) Marshal(v interface{}, opts ...codec.Option) ([]byte, error) { @@ -30,8 +42,10 @@ func (c *protoCodec) Marshal(v interface{}, opts ...codec.Option) ([]byte, error o(&options) } - if nv, nerr := rutil.StructFieldByTag(v, options.TagName, flattenTag); nerr == nil { - v = nv + if options.Flatten { + if nv, nerr := rutil.StructFieldByTag(v, options.TagName, "flatten"); nerr == nil { + v = nv + } } switch m := v.(type) { @@ -39,13 +53,17 @@ func (c *protoCodec) Marshal(v interface{}, opts ...codec.Option) ([]byte, error return m.Data, nil case *pb.Frame: return m.Data, nil - } - - if _, ok := v.(proto.Message); !ok { + case proto.Message: + marshalOptions := DefaultMarshalOptions + if options.Context != nil { + if f, ok := options.Context.Value(marshalOptionsKey{}).(proto.MarshalOptions); ok { + marshalOptions = f + } + } + return marshalOptions.Marshal(m) + default: return nil, codec.ErrInvalidMessage } - - return proto.Marshal(v.(proto.Message)) } func (c *protoCodec) Unmarshal(d []byte, v interface{}, opts ...codec.Option) error { @@ -58,8 +76,10 @@ func (c *protoCodec) Unmarshal(d []byte, v interface{}, opts ...codec.Option) er o(&options) } - if nv, nerr := rutil.StructFieldByTag(v, options.TagName, flattenTag); nerr == nil { - v = nv + if options.Flatten { + if nv, nerr := rutil.StructFieldByTag(v, options.TagName, "flatten"); nerr == nil { + v = nv + } } switch m := v.(type) { @@ -69,46 +89,17 @@ func (c *protoCodec) Unmarshal(d []byte, v interface{}, opts ...codec.Option) er case *pb.Frame: m.Data = d return nil - } - - if _, ok := v.(proto.Message); !ok { + case proto.Message: + unmarshalOptions := DefaultUnmarshalOptions + if options.Context != nil { + if f, ok := options.Context.Value(marshalOptionsKey{}).(proto.UnmarshalOptions); ok { + unmarshalOptions = f + } + } + return unmarshalOptions.Unmarshal(d, m) + default: return codec.ErrInvalidMessage } - - return proto.Unmarshal(d, v.(proto.Message)) -} - -func (c *protoCodec) ReadHeader(conn io.Reader, m *codec.Message, t codec.MessageType) error { - return nil -} - -func (c *protoCodec) ReadBody(conn io.Reader, v interface{}) error { - if v == nil { - return nil - } - - buf, err := io.ReadAll(conn) - if err != nil { - return err - } else if len(buf) == 0 { - return nil - } - return c.Unmarshal(buf, v) -} - -func (c *protoCodec) Write(conn io.Writer, m *codec.Message, v interface{}) error { - if v == nil { - return nil - } - - buf, err := c.Marshal(v) - if err != nil { - return err - } else if len(buf) == 0 { - return codec.ErrInvalidMessage - } - _, err = conn.Write(buf) - return err } func (c *protoCodec) String() string { @@ -118,3 +109,81 @@ func (c *protoCodec) String() string { func NewCodec(opts ...codec.Option) codec.Codec { return &protoCodec{opts: codec.NewOptions(opts...)} } + +func (c *protoCodecV2) Marshal(d []byte, v interface{}, opts ...codec.Option) ([]byte, error) { + if v == nil { + return nil, nil + } + + options := c.opts + for _, o := range opts { + o(&options) + } + + if options.Flatten { + if nv, nerr := rutil.StructFieldByTag(v, options.TagName, "flatten"); nerr == nil { + v = nv + } + } + + switch m := v.(type) { + case *codec.Frame: + return m.Data, nil + case *pb.Frame: + return m.Data, nil + case proto.Message: + marshalOptions := DefaultMarshalOptions + if options.Context != nil { + if f, ok := options.Context.Value(marshalOptionsKey{}).(proto.MarshalOptions); ok { + marshalOptions = f + } + } + return marshalOptions.MarshalAppend(d, m) + default: + return nil, codec.ErrInvalidMessage + } +} + +func (c *protoCodecV2) Unmarshal(d []byte, v interface{}, opts ...codec.Option) error { + if v == nil || len(d) == 0 { + return nil + } + + options := c.opts + for _, o := range opts { + o(&options) + } + + if options.Flatten { + if nv, nerr := rutil.StructFieldByTag(v, options.TagName, "flatten"); nerr == nil { + v = nv + } + } + + switch m := v.(type) { + case *codec.Frame: + m.Data = d + return nil + case *pb.Frame: + m.Data = d + return nil + case proto.Message: + unmarshalOptions := DefaultUnmarshalOptions + if options.Context != nil { + if f, ok := options.Context.Value(marshalOptionsKey{}).(proto.UnmarshalOptions); ok { + unmarshalOptions = f + } + } + return unmarshalOptions.Unmarshal(d, m) + default: + return codec.ErrInvalidMessage + } +} + +func (c *protoCodecV2) String() string { + return "proto" +} + +func NewCodecV2(opts ...codec.Option) codec.CodecV2 { + return &protoCodecV2{opts: codec.NewOptions(opts...)} +}