From 2b7fc788727c835298e1dd82f212693797727cfd Mon Sep 17 00:00:00 2001 From: Gorbunov Kirill Andreevich Date: Thu, 21 Nov 2024 15:03:07 +0300 Subject: [PATCH] #1-add protoset --- .gitignore | 1 + cmd/servicechecker/main.go | 50 ++++------------- go.mod | 4 ++ go.sum | 8 +++ pkg/protoset/protoset.go | 100 ++++++++++++++++++++++++++++++++++ pkg/protoset/protoset_test.go | 50 +++++++++++++++++ 6 files changed, 175 insertions(+), 38 deletions(-) create mode 100644 pkg/protoset/protoset.go create mode 100644 pkg/protoset/protoset_test.go diff --git a/.gitignore b/.gitignore index c523002..0c52f0b 100644 --- a/.gitignore +++ b/.gitignore @@ -25,3 +25,4 @@ config.yaml servicechecker *.protoset authn.yaml +.idea \ No newline at end of file diff --git a/cmd/servicechecker/main.go b/cmd/servicechecker/main.go index 9be32d7..90bdb66 100644 --- a/cmd/servicechecker/main.go +++ b/cmd/servicechecker/main.go @@ -29,11 +29,8 @@ import ( "go.unistack.org/servicechecker/pkg/config" "go.unistack.org/servicechecker/pkg/grpcconn" "go.unistack.org/servicechecker/pkg/httpconn" + "go.unistack.org/servicechecker/pkg/protoset" "go.unistack.org/servicechecker/pkg/scheduler" - "google.golang.org/protobuf/reflect/protodesc" - "google.golang.org/protobuf/reflect/protoreflect" - "google.golang.org/protobuf/types/descriptorpb" - "google.golang.org/protobuf/types/dynamicpb" ) var ( @@ -133,7 +130,7 @@ func main() { client.Codec("application/grpc+json", jsonpbcodec.NewCodec()), client.ContentType("application/grpc"), client.Retries(0), - // client.TLSConfig(&tls.Config{InsecureSkipVerify: true}), + // client.TLSConfig(&tls.Config{InsecureSkipVerify: true}), ) if err = gcli.Init(); err != nil { l.Fatal(ctx, "failed to init grpc client", err) @@ -146,7 +143,7 @@ func main() { client.Codec("application/json", jsonpbcodec.NewCodec()), client.ContentType("application/json"), client.Retries(0), - // client.TLSConfig(&tls.Config{InsecureSkipVerify: true}), + // client.TLSConfig(&tls.Config{InsecureSkipVerify: true}), ) if err = hcli.Init(); err != nil { l.Fatal(ctx, "failed to init http client", err) @@ -179,10 +176,11 @@ func main() { var fn any var args []any + p := protoset.NewProtoSet(l) switch { case task.GRPC != nil: - fn, args, err = newGRPCTask(ctx, l, mtr, check.Name, task) + fn, args, err = newGRPCTask(ctx, l, mtr, p, check.Name, task) case task.HTTP != nil: fn, args, err = newHTTPTask(ctx, l, mtr, check.Name, task) } @@ -291,7 +289,7 @@ func newHTTPTask(ctx context.Context, l logger.Logger, m meter.Meter, check stri return fn, nil, nil } -func newGRPCTask(ctx context.Context, l logger.Logger, m meter.Meter, check string, task *config.TaskConfig) (any, []any, error) { +func newGRPCTask(ctx context.Context, l logger.Logger, m meter.Meter, p *protoset.ProtoSet, check string, task *config.TaskConfig) (any, []any, error) { var err error c, ok := clients["grpc"] @@ -307,49 +305,25 @@ func newGRPCTask(ctx context.Context, l logger.Logger, m meter.Meter, check stri var labels []string if task.GRPC.Protoset != "" { + pkg, svc, mth := grpcconn.ServiceMethod(task.GRPC.Endpoint) + protosetBuf, err := os.ReadFile(task.GRPC.Protoset) if err != nil { l.Error(ctx, "failed to unmarshal protoset file", err) return nil, nil, err } - fdset := &descriptorpb.FileDescriptorSet{} - if err = protocodec.NewCodec().Unmarshal(protosetBuf, fdset); err != nil { - l.Error(ctx, "failed to unmarshal protoset file", err) + if err = p.AddProtoset(ctx, task.GRPC.Addr, svc, protosetBuf); err != nil { + l.Error(ctx, "failed add protoset", err) return nil, nil, err } - pfileoptions := protodesc.FileOptions{AllowUnresolvable: true} - pfiles, err := pfileoptions.NewFiles(fdset) + req, rsp, err = p.GetMessage(ctx, task.GRPC.Addr, pkg, svc, mth) if err != nil { - l.Error(ctx, "failed to use protoset file", err) + l.Error(ctx, "failed get req, rsp from protoset", err) return nil, nil, err } - pkg, svc, mth := grpcconn.ServiceMethod(task.GRPC.Endpoint) - pdesc, err := pfiles.FindDescriptorByName(protoreflect.FullName(pkg + "." + svc)) - if err != nil { - l.Error(ctx, "failed to find service "+pkg+"."+svc) - return nil, nil, err - } - - sdesc, ok := pdesc.(protoreflect.ServiceDescriptor) - if !ok { - err = fmt.Errorf("failed to find service " + pkg + "." + svc) - l.Error(ctx, "unable to find service in protoset", err) - return nil, nil, err - } - - mdesc := sdesc.Methods().ByName(protoreflect.Name(mth)) - if mdesc == nil { - err = fmt.Errorf("unknown method " + mth) - l.Error(ctx, "failed to find method", err) - return nil, nil, err - } - - req = dynamicpb.NewMessageType(mdesc.Input()).New() - rsp = dynamicpb.NewMessageType(mdesc.Output()).New() - if err = jsonpbcodec.NewCodec().Unmarshal([]byte(task.GRPC.Data), req); err != nil { l.Error(ctx, "failed to unmarshal", err) return nil, nil, err diff --git a/go.mod b/go.mod index 8b63217..3094e5c 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( github.com/go-co-op/gocron/v2 v2.12.3 github.com/google/gnostic v0.7.0 github.com/google/uuid v1.6.0 + github.com/smartystreets/goconvey v1.8.1 go.unistack.org/micro-client-grpc/v3 v3.11.10 go.unistack.org/micro-client-http/v3 v3.9.14 go.unistack.org/micro-codec-json/v3 v3.10.1 @@ -21,8 +22,11 @@ require ( require ( github.com/google/gnostic-models v0.6.9-0.20230804172637-c7be7c783f49 // indirect + github.com/gopherjs/gopherjs v1.17.2 // indirect github.com/jonboulle/clockwork v0.4.0 // indirect + github.com/jtolds/gls v4.20.0+incompatible // indirect github.com/robfig/cron/v3 v3.0.1 // indirect + github.com/smarty/assertions v1.15.0 // indirect github.com/valyala/fastrand v1.1.0 // indirect github.com/valyala/histogram v1.2.0 // indirect go.unistack.org/metrics v0.0.1 // indirect diff --git a/go.sum b/go.sum index 4082fbf..7b09e37 100644 --- a/go.sum +++ b/go.sum @@ -782,6 +782,8 @@ github.com/googleapis/gax-go/v2 v2.7.0/go.mod h1:TEop28CZZQ2y+c0VxMUmu1lV+fQx57Q github.com/googleapis/gax-go/v2 v2.7.1/go.mod h1:4orTrqY6hXxxaUL4LHIPl6lGo8vAE38/qKbhSAKP6QI= github.com/googleapis/go-type-adapters v1.0.0/go.mod h1:zHW75FOG2aur7gAO2B+MLby+cLsWGBF62rFAi7WjWO4= github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= +github.com/gopherjs/gopherjs v1.17.2 h1:fQnZVsXk8uxXIStYb0N4bGk7jeyTalG/wsZjQ25dO0g= +github.com/gopherjs/gopherjs v1.17.2/go.mod h1:pRRIvn/QzFLrKfvEz3qUuEhtE/zLCWfreZ6J5gM2i+k= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0/go.mod h1:hgWBS7lorOAVIJEQMi4ZsPv9hVvWI6+ch50m39Pf2Ks= github.com/grpc-ecosystem/grpc-gateway/v2 v2.11.3/go.mod h1:o//XUCC/F+yRGJoPO/VU0GSB0f8Nhgmxx0VIRUvaC0w= @@ -794,6 +796,8 @@ github.com/jonboulle/clockwork v0.4.0 h1:p4Cf1aMWXnXAUh8lVfewRBx1zaTSYKrKMF2g3ST github.com/jonboulle/clockwork v0.4.0/go.mod h1:xgRqUGwRcjKCO1vbZUEtSLrqKoPSsUpK7fnezOII0kc= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= +github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/jung-kurt/gofpdf v1.0.0/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E/uU8ce6rXgefFLlgrJj/GYY22cpxn+r32jIOes= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= @@ -845,6 +849,10 @@ github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/f github.com/ruudk/golang-pdf417 v0.0.0-20181029194003-1af4ab5afa58/go.mod h1:6lfFZQK844Gfx8o5WFuvpxWRwnSoipWe/p622j1v06w= github.com/ruudk/golang-pdf417 v0.0.0-20201230142125-a7e3863a1245/go.mod h1:pQAZKsJ8yyVxGRWYNEm9oFB8ieLgKFnamEyDmSA0BRk= github.com/silas/dag v0.0.0-20220518035006-a7e85ada93c5/go.mod h1:7RTUFBdIRC9nZ7/3RyRNH1bdqIShrDejd1YbLwgPS+I= +github.com/smarty/assertions v1.15.0 h1:cR//PqUBUiQRakZWqBiFFQ9wb8emQGDb0HeGdqGByCY= +github.com/smarty/assertions v1.15.0/go.mod h1:yABtdzeQs6l1brC900WlRNwj6ZR55d7B+E8C6HtKdec= +github.com/smartystreets/goconvey v1.8.1 h1:qGjIddxOk4grTu9JPOU31tVfq3cNdBlNa5sSznIX1xY= +github.com/smartystreets/goconvey v1.8.1/go.mod h1:+/u4qLyY6x1jReYOp7GOM2FSt8aP9CzCZL03bI28W60= github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.3.3/go.mod h1:5KUK8ByomD5Ti5Artl0RtHeI5pTF7MIDuXL3yY520V4= github.com/spf13/afero v1.6.0/go.mod h1:Ai8FlHk4v/PARR026UzYexafAt9roJ7LcLMAmO6Z93I= diff --git a/pkg/protoset/protoset.go b/pkg/protoset/protoset.go new file mode 100644 index 0000000..c33548f --- /dev/null +++ b/pkg/protoset/protoset.go @@ -0,0 +1,100 @@ +package protoset + +import ( + "context" + "errors" + "fmt" + "sync" + + protocodec "go.unistack.org/micro-codec-proto/v3" + "go.unistack.org/micro/v3/logger" + "google.golang.org/protobuf/reflect/protodesc" + "google.golang.org/protobuf/reflect/protoreflect" + "google.golang.org/protobuf/reflect/protoregistry" + "google.golang.org/protobuf/types/descriptorpb" + "google.golang.org/protobuf/types/dynamicpb" +) + +var errNotFound = errors.New("file descriptor not found") + +type ProtoSet struct { + mu sync.Mutex + files map[string]*protoregistry.Files + log logger.Logger +} + +func NewProtoSet(log logger.Logger) *ProtoSet { + return &ProtoSet{ + mu: sync.Mutex{}, + files: make(map[string]*protoregistry.Files, 0), + log: log, + } +} + +func (p *ProtoSet) GetMessage(ctx context.Context, addr, pkg, svc, mth string) (protoreflect.Message, protoreflect.Message, error) { + p.log.Debug(ctx, "start of GetMessage") + if addr == "" || svc == "" || mth == "" { + p.log.Error(ctx, "protoset: empty addr or service param") + return nil, nil, errors.New("addr or service name is empty") + } + p.mu.Lock() + pfile, ok := p.files[addr+"|"+svc] + p.mu.Unlock() + if !ok || pfile == nil { + p.log.Error(ctx, "protoset: file desc not found") + return nil, nil, errNotFound + } + + pdesc, err := pfile.FindDescriptorByName(protoreflect.FullName(pkg + "." + svc)) + if err != nil { + p.log.Error(ctx, "failed to find service "+pkg+"."+svc) + return nil, nil, err + } + + sdesc, ok := pdesc.(protoreflect.ServiceDescriptor) + if !ok { + err = fmt.Errorf("failed to find service " + pkg + "." + svc) + p.log.Error(ctx, "unable to find service in protoset", err) + return nil, nil, err + } + + mdesc := sdesc.Methods().ByName(protoreflect.Name(mth)) + if mdesc == nil { + err = fmt.Errorf("unknown method " + mth) + p.log.Error(ctx, "failed to find method", err) + return nil, nil, err + } + + req := dynamicpb.NewMessageType(mdesc.Input()).New() + rsp := dynamicpb.NewMessageType(mdesc.Output()).New() + + return req, rsp, nil +} + +func (p *ProtoSet) AddProtoset(ctx context.Context, addr, svc string, data []byte) error { + p.log.Debug(ctx, "start of AddProtoset") + + fdset := &descriptorpb.FileDescriptorSet{} + if err := protocodec.NewCodec().Unmarshal(data, fdset); err != nil { + p.log.Error(ctx, "failed to unmarshal protoset file", err) + return err + } + + pfileoptions := protodesc.FileOptions{AllowUnresolvable: true} + pfiles, err := pfileoptions.NewFiles(fdset) + if err != nil { + p.log.Error(ctx, "failed to use protoset file", err) + return err + } + + p.mu.Lock() + p.files[addr+"|"+svc] = pfiles + p.mu.Unlock() + + return nil +} + +func (p *ProtoSet) AddReflection(ctx context.Context, service string, addr string) error { + p.log.Debug(ctx, "start of AddReflection") + return nil +} diff --git a/pkg/protoset/protoset_test.go b/pkg/protoset/protoset_test.go new file mode 100644 index 0000000..5d5fcfa --- /dev/null +++ b/pkg/protoset/protoset_test.go @@ -0,0 +1,50 @@ +package protoset + +import ( + "context" + "fmt" + "os" + "testing" + + . "github.com/smartystreets/goconvey/convey" + "go.unistack.org/micro/v3/logger" +) + +func TestProtoSet(t *testing.T) { + Convey("test1", t, func() { + ctx := context.Background() + p := NewProtoSet(logger.DefaultLogger) + data, err := os.ReadFile("/Users/kgorbunov/GolandProjects/src/card-proto/card-grpc.protoset") + So(err, ShouldBeNil) + + err = p.AddProtoset(ctx, "localhost:9090", "CardService", data) + So(err, ShouldBeNil) + + req, rsp, err := p.GetMessage(ctx, "localhost:9090", "card_proto", "CardService", "GetCardList") + So(err, ShouldBeNil) + fmt.Printf("req: %v, rsp: %v", req, rsp) + }) + + Convey("test2-bad", t, func() { + ctx := context.Background() + p := NewProtoSet(logger.DefaultLogger) + data, err := os.ReadFile("/Users/kgorbunov/GolandProjects/src/card-proto/card-grpc.protoset") + So(err, ShouldBeNil) + + err = p.AddProtoset(ctx, "localhost:9090", "CardService", data) + So(err, ShouldBeNil) + + req, rsp, err := p.GetMessage(ctx, "localhost:9090", "card_proto", "Card", "GetCardList") + So(err, ShouldBeError) + fmt.Printf("req: %v, rsp: %v", req, rsp) + }) + + Convey("test2-not-found", t, func() { + ctx := context.Background() + p := NewProtoSet(logger.DefaultLogger) + + req, rsp, err := p.GetMessage(ctx, "localhost:9090", "card_proto", "CardService", "GetCardList") + So(err, ShouldEqual, errNotFound) + fmt.Printf("req: %v, rsp: %v", req, rsp) + }) +}