From 29cccd0b4ab6132f1a738ec63ddd35bf9c8e2659 Mon Sep 17 00:00:00 2001 From: Asim Aslam Date: Thu, 9 Apr 2020 14:10:17 +0100 Subject: [PATCH 01/44] minor tweak add log line to proxy and basic auth provider by default (#1513) --- auth/default.go | 13 ++++++++++++- proxy/mucp/mucp.go | 2 +- 2 files changed, 13 insertions(+), 2 deletions(-) diff --git a/auth/default.go b/auth/default.go index 358cf099..b9160a50 100644 --- a/auth/default.go +++ b/auth/default.go @@ -2,6 +2,7 @@ package auth import ( "github.com/google/uuid" + "github.com/micro/go-micro/v2/auth/provider/basic" ) var ( @@ -9,7 +10,17 @@ var ( ) func NewAuth(opts ...Option) Auth { - return &noop{} + options := Options{ + Provider: basic.NewProvider(), + } + + for _, o := range opts { + o(&options) + } + + return &noop{ + opts: options, + } } type noop struct { diff --git a/proxy/mucp/mucp.go b/proxy/mucp/mucp.go index da901c13..e7c69aa2 100644 --- a/proxy/mucp/mucp.go +++ b/proxy/mucp/mucp.go @@ -206,7 +206,7 @@ func (p *Proxy) cacheRoutes(service string) ([]router.Route, error) { results, err := p.Router.Lookup(router.QueryService(service)) if err != nil { // assumption that we're ok with stale routes - + logger.Debugf("Failed to lookup route for %s: %v", service, err) // otherwise return the error return nil, err } From 77f0abb0ba550948abbe793aed54460e0703caf8 Mon Sep 17 00:00:00 2001 From: Janos Dobronszki Date: Thu, 9 Apr 2020 16:44:39 +0200 Subject: [PATCH 02/44] Enabling micro run for subfolders (#1510) * Enabling micro run for subfolders * Use source instead of os.Args[2] * Works now * PR comments * WorkDir -> Dir --- runtime/local/process/os/os.go | 4 ++++ runtime/local/process/process.go | 2 ++ runtime/service.go | 1 + 3 files changed, 7 insertions(+) diff --git a/runtime/local/process/os/os.go b/runtime/local/process/os/os.go index f1825e97..a35d0b79 100644 --- a/runtime/local/process/os/os.go +++ b/runtime/local/process/os/os.go @@ -15,13 +15,17 @@ import ( func (p *Process) Exec(exe *process.Executable) error { cmd := exec.Command(exe.Package.Path) + cmd.Dir = exe.Dir return cmd.Run() } func (p *Process) Fork(exe *process.Executable) (*process.PID, error) { // create command cmd := exec.Command(exe.Package.Path, exe.Args...) + + cmd.Dir = exe.Dir // set env vars + cmd.Env = append(cmd.Env, os.Environ()...) cmd.Env = append(cmd.Env, exe.Env...) // create process group diff --git a/runtime/local/process/process.go b/runtime/local/process/process.go index 4449d294..ffeae63e 100644 --- a/runtime/local/process/process.go +++ b/runtime/local/process/process.go @@ -26,6 +26,8 @@ type Executable struct { Env []string // Args to pass Args []string + // Initial working directory + Dir string } // PID is the running process diff --git a/runtime/service.go b/runtime/service.go index 9c6bdc16..eb6c4154 100644 --- a/runtime/service.go +++ b/runtime/service.go @@ -55,6 +55,7 @@ func newService(s *Service, c CreateOptions) *service { }, Env: c.Env, Args: args, + Dir: s.Source, }, closed: make(chan bool), output: c.Output, From 0a27a0818410abe949e96d858b308e99bee532d2 Mon Sep 17 00:00:00 2001 From: Jake Sanders Date: Thu, 9 Apr 2020 16:37:32 +0100 Subject: [PATCH 03/44] Add Databases and Tables endpoints to store RPC proto (#1515) * Add Databases and Tables to store RPC * add Database to TablesRequest --- store/service/proto/store.pb.go | 471 +++++++++++--------------- store/service/proto/store.pb.micro.go | 36 +- store/service/proto/store.proto | 16 + 3 files changed, 256 insertions(+), 267 deletions(-) diff --git a/store/service/proto/store.pb.go b/store/service/proto/store.pb.go index 177e664d..bcab9640 100644 --- a/store/service/proto/store.pb.go +++ b/store/service/proto/store.pb.go @@ -1,15 +1,11 @@ // Code generated by protoc-gen-go. DO NOT EDIT. -// source: store/service/proto/store.proto +// source: store.proto package go_micro_store import ( - context "context" fmt "fmt" proto "github.com/golang/protobuf/proto" - grpc "google.golang.org/grpc" - codes "google.golang.org/grpc/codes" - status "google.golang.org/grpc/status" math "math" ) @@ -40,7 +36,7 @@ func (m *Record) Reset() { *m = Record{} } func (m *Record) String() string { return proto.CompactTextString(m) } func (*Record) ProtoMessage() {} func (*Record) Descriptor() ([]byte, []int) { - return fileDescriptor_1ba364858f5c3cdb, []int{0} + return fileDescriptor_98bbca36ef968dfc, []int{0} } func (m *Record) XXX_Unmarshal(b []byte) error { @@ -96,7 +92,7 @@ func (m *ReadOptions) Reset() { *m = ReadOptions{} } func (m *ReadOptions) String() string { return proto.CompactTextString(m) } func (*ReadOptions) ProtoMessage() {} func (*ReadOptions) Descriptor() ([]byte, []int) { - return fileDescriptor_1ba364858f5c3cdb, []int{1} + return fileDescriptor_98bbca36ef968dfc, []int{1} } func (m *ReadOptions) XXX_Unmarshal(b []byte) error { @@ -157,7 +153,7 @@ func (m *ReadRequest) Reset() { *m = ReadRequest{} } func (m *ReadRequest) String() string { return proto.CompactTextString(m) } func (*ReadRequest) ProtoMessage() {} func (*ReadRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_1ba364858f5c3cdb, []int{2} + return fileDescriptor_98bbca36ef968dfc, []int{2} } func (m *ReadRequest) XXX_Unmarshal(b []byte) error { @@ -203,7 +199,7 @@ func (m *ReadResponse) Reset() { *m = ReadResponse{} } func (m *ReadResponse) String() string { return proto.CompactTextString(m) } func (*ReadResponse) ProtoMessage() {} func (*ReadResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_1ba364858f5c3cdb, []int{3} + return fileDescriptor_98bbca36ef968dfc, []int{3} } func (m *ReadResponse) XXX_Unmarshal(b []byte) error { @@ -245,7 +241,7 @@ func (m *WriteOptions) Reset() { *m = WriteOptions{} } func (m *WriteOptions) String() string { return proto.CompactTextString(m) } func (*WriteOptions) ProtoMessage() {} func (*WriteOptions) Descriptor() ([]byte, []int) { - return fileDescriptor_1ba364858f5c3cdb, []int{4} + return fileDescriptor_98bbca36ef968dfc, []int{4} } func (m *WriteOptions) XXX_Unmarshal(b []byte) error { @@ -292,7 +288,7 @@ func (m *WriteRequest) Reset() { *m = WriteRequest{} } func (m *WriteRequest) String() string { return proto.CompactTextString(m) } func (*WriteRequest) ProtoMessage() {} func (*WriteRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_1ba364858f5c3cdb, []int{5} + return fileDescriptor_98bbca36ef968dfc, []int{5} } func (m *WriteRequest) XXX_Unmarshal(b []byte) error { @@ -337,7 +333,7 @@ func (m *WriteResponse) Reset() { *m = WriteResponse{} } func (m *WriteResponse) String() string { return proto.CompactTextString(m) } func (*WriteResponse) ProtoMessage() {} func (*WriteResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_1ba364858f5c3cdb, []int{6} + return fileDescriptor_98bbca36ef968dfc, []int{6} } func (m *WriteResponse) XXX_Unmarshal(b []byte) error { @@ -368,7 +364,7 @@ func (m *DeleteOptions) Reset() { *m = DeleteOptions{} } func (m *DeleteOptions) String() string { return proto.CompactTextString(m) } func (*DeleteOptions) ProtoMessage() {} func (*DeleteOptions) Descriptor() ([]byte, []int) { - return fileDescriptor_1ba364858f5c3cdb, []int{7} + return fileDescriptor_98bbca36ef968dfc, []int{7} } func (m *DeleteOptions) XXX_Unmarshal(b []byte) error { @@ -401,7 +397,7 @@ func (m *DeleteRequest) Reset() { *m = DeleteRequest{} } func (m *DeleteRequest) String() string { return proto.CompactTextString(m) } func (*DeleteRequest) ProtoMessage() {} func (*DeleteRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_1ba364858f5c3cdb, []int{8} + return fileDescriptor_98bbca36ef968dfc, []int{8} } func (m *DeleteRequest) XXX_Unmarshal(b []byte) error { @@ -446,7 +442,7 @@ func (m *DeleteResponse) Reset() { *m = DeleteResponse{} } func (m *DeleteResponse) String() string { return proto.CompactTextString(m) } func (*DeleteResponse) ProtoMessage() {} func (*DeleteResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_1ba364858f5c3cdb, []int{9} + return fileDescriptor_98bbca36ef968dfc, []int{9} } func (m *DeleteResponse) XXX_Unmarshal(b []byte) error { @@ -481,7 +477,7 @@ func (m *ListOptions) Reset() { *m = ListOptions{} } func (m *ListOptions) String() string { return proto.CompactTextString(m) } func (*ListOptions) ProtoMessage() {} func (*ListOptions) Descriptor() ([]byte, []int) { - return fileDescriptor_1ba364858f5c3cdb, []int{10} + return fileDescriptor_98bbca36ef968dfc, []int{10} } func (m *ListOptions) XXX_Unmarshal(b []byte) error { @@ -541,7 +537,7 @@ func (m *ListRequest) Reset() { *m = ListRequest{} } func (m *ListRequest) String() string { return proto.CompactTextString(m) } func (*ListRequest) ProtoMessage() {} func (*ListRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_1ba364858f5c3cdb, []int{11} + return fileDescriptor_98bbca36ef968dfc, []int{11} } func (m *ListRequest) XXX_Unmarshal(b []byte) error { @@ -580,7 +576,7 @@ func (m *ListResponse) Reset() { *m = ListResponse{} } func (m *ListResponse) String() string { return proto.CompactTextString(m) } func (*ListResponse) ProtoMessage() {} func (*ListResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_1ba364858f5c3cdb, []int{12} + return fileDescriptor_98bbca36ef968dfc, []int{12} } func (m *ListResponse) XXX_Unmarshal(b []byte) error { @@ -608,6 +604,154 @@ func (m *ListResponse) GetKeys() []string { return nil } +type DatabasesRequest struct { + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *DatabasesRequest) Reset() { *m = DatabasesRequest{} } +func (m *DatabasesRequest) String() string { return proto.CompactTextString(m) } +func (*DatabasesRequest) ProtoMessage() {} +func (*DatabasesRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_98bbca36ef968dfc, []int{13} +} + +func (m *DatabasesRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_DatabasesRequest.Unmarshal(m, b) +} +func (m *DatabasesRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_DatabasesRequest.Marshal(b, m, deterministic) +} +func (m *DatabasesRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_DatabasesRequest.Merge(m, src) +} +func (m *DatabasesRequest) XXX_Size() int { + return xxx_messageInfo_DatabasesRequest.Size(m) +} +func (m *DatabasesRequest) XXX_DiscardUnknown() { + xxx_messageInfo_DatabasesRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_DatabasesRequest proto.InternalMessageInfo + +type DatabasesResponse struct { + Databases []string `protobuf:"bytes,1,rep,name=databases,proto3" json:"databases,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *DatabasesResponse) Reset() { *m = DatabasesResponse{} } +func (m *DatabasesResponse) String() string { return proto.CompactTextString(m) } +func (*DatabasesResponse) ProtoMessage() {} +func (*DatabasesResponse) Descriptor() ([]byte, []int) { + return fileDescriptor_98bbca36ef968dfc, []int{14} +} + +func (m *DatabasesResponse) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_DatabasesResponse.Unmarshal(m, b) +} +func (m *DatabasesResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_DatabasesResponse.Marshal(b, m, deterministic) +} +func (m *DatabasesResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_DatabasesResponse.Merge(m, src) +} +func (m *DatabasesResponse) XXX_Size() int { + return xxx_messageInfo_DatabasesResponse.Size(m) +} +func (m *DatabasesResponse) XXX_DiscardUnknown() { + xxx_messageInfo_DatabasesResponse.DiscardUnknown(m) +} + +var xxx_messageInfo_DatabasesResponse proto.InternalMessageInfo + +func (m *DatabasesResponse) GetDatabases() []string { + if m != nil { + return m.Databases + } + return nil +} + +type TablesRequest struct { + Database string `protobuf:"bytes,1,opt,name=database,proto3" json:"database,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *TablesRequest) Reset() { *m = TablesRequest{} } +func (m *TablesRequest) String() string { return proto.CompactTextString(m) } +func (*TablesRequest) ProtoMessage() {} +func (*TablesRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_98bbca36ef968dfc, []int{15} +} + +func (m *TablesRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_TablesRequest.Unmarshal(m, b) +} +func (m *TablesRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_TablesRequest.Marshal(b, m, deterministic) +} +func (m *TablesRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_TablesRequest.Merge(m, src) +} +func (m *TablesRequest) XXX_Size() int { + return xxx_messageInfo_TablesRequest.Size(m) +} +func (m *TablesRequest) XXX_DiscardUnknown() { + xxx_messageInfo_TablesRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_TablesRequest proto.InternalMessageInfo + +func (m *TablesRequest) GetDatabase() string { + if m != nil { + return m.Database + } + return "" +} + +type TablesResponse struct { + Tables []string `protobuf:"bytes,1,rep,name=tables,proto3" json:"tables,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *TablesResponse) Reset() { *m = TablesResponse{} } +func (m *TablesResponse) String() string { return proto.CompactTextString(m) } +func (*TablesResponse) ProtoMessage() {} +func (*TablesResponse) Descriptor() ([]byte, []int) { + return fileDescriptor_98bbca36ef968dfc, []int{16} +} + +func (m *TablesResponse) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_TablesResponse.Unmarshal(m, b) +} +func (m *TablesResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_TablesResponse.Marshal(b, m, deterministic) +} +func (m *TablesResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_TablesResponse.Merge(m, src) +} +func (m *TablesResponse) XXX_Size() int { + return xxx_messageInfo_TablesResponse.Size(m) +} +func (m *TablesResponse) XXX_DiscardUnknown() { + xxx_messageInfo_TablesResponse.DiscardUnknown(m) +} + +var xxx_messageInfo_TablesResponse proto.InternalMessageInfo + +func (m *TablesResponse) GetTables() []string { + if m != nil { + return m.Tables + } + return nil +} + func init() { proto.RegisterType((*Record)(nil), "go.micro.store.Record") proto.RegisterType((*ReadOptions)(nil), "go.micro.store.ReadOptions") @@ -622,256 +766,51 @@ func init() { proto.RegisterType((*ListOptions)(nil), "go.micro.store.ListOptions") proto.RegisterType((*ListRequest)(nil), "go.micro.store.ListRequest") proto.RegisterType((*ListResponse)(nil), "go.micro.store.ListResponse") + proto.RegisterType((*DatabasesRequest)(nil), "go.micro.store.DatabasesRequest") + proto.RegisterType((*DatabasesResponse)(nil), "go.micro.store.DatabasesResponse") + proto.RegisterType((*TablesRequest)(nil), "go.micro.store.TablesRequest") + proto.RegisterType((*TablesResponse)(nil), "go.micro.store.TablesResponse") } -func init() { proto.RegisterFile("store/service/proto/store.proto", fileDescriptor_1ba364858f5c3cdb) } - -var fileDescriptor_1ba364858f5c3cdb = []byte{ - // 474 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x54, 0x5d, 0x6f, 0xd3, 0x30, - 0x14, 0x9d, 0x9b, 0x34, 0x5b, 0x6f, 0xcb, 0xa8, 0x2c, 0x34, 0x45, 0xb0, 0x41, 0xe5, 0xa7, 0x3c, - 0xa5, 0x53, 0x11, 0x1f, 0x8f, 0x48, 0x0c, 0x04, 0x08, 0x09, 0xc9, 0x48, 0x20, 0xf1, 0x36, 0xba, - 0x5b, 0x64, 0xb5, 0x9b, 0x83, 0xed, 0x56, 0xeb, 0x1f, 0xe2, 0x77, 0x22, 0x7f, 0xb5, 0x69, 0x48, - 0x5e, 0x78, 0xf3, 0xbd, 0xbe, 0x39, 0xe7, 0x9e, 0xe3, 0xa3, 0xc0, 0x33, 0x6d, 0xa4, 0xc2, 0xa9, - 0x46, 0xb5, 0x11, 0x73, 0x9c, 0x56, 0x4a, 0x1a, 0x39, 0x75, 0xbd, 0xd2, 0x9d, 0xe9, 0xe9, 0x2f, - 0x59, 0xde, 0x8a, 0xb9, 0x92, 0xa5, 0xeb, 0xb2, 0x0f, 0x90, 0x71, 0x9c, 0x4b, 0x75, 0x43, 0xc7, - 0x90, 0x2c, 0x71, 0x9b, 0x93, 0x09, 0x29, 0x06, 0xdc, 0x1e, 0xe9, 0x23, 0xe8, 0x6f, 0xae, 0x57, - 0x6b, 0xcc, 0x7b, 0x13, 0x52, 0x8c, 0xb8, 0x2f, 0xe8, 0x19, 0x64, 0x78, 0x5f, 0x09, 0xb5, 0xcd, - 0x93, 0x09, 0x29, 0x12, 0x1e, 0x2a, 0xb6, 0x84, 0x21, 0xc7, 0xeb, 0x9b, 0x2f, 0x95, 0x11, 0xf2, - 0x4e, 0xdb, 0xb1, 0x4a, 0xe1, 0x42, 0xdc, 0x3b, 0xc4, 0x13, 0x1e, 0x2a, 0xdb, 0xd7, 0xeb, 0x85, - 0xed, 0xf7, 0x7c, 0xdf, 0x57, 0x96, 0x6c, 0x25, 0x6e, 0x85, 0x71, 0xa8, 0x29, 0xf7, 0x85, 0x9d, - 0x96, 0x8b, 0x85, 0x46, 0x93, 0xa7, 0xae, 0x1d, 0x2a, 0xf6, 0xcd, 0x93, 0x71, 0xfc, 0xbd, 0x46, - 0x6d, 0x5a, 0x76, 0x7f, 0x01, 0xc7, 0xd2, 0x6f, 0xe2, 0x78, 0x86, 0xb3, 0x27, 0xe5, 0xa1, 0xf2, - 0xb2, 0xb6, 0x2c, 0x8f, 0xb3, 0xec, 0x0d, 0x8c, 0x3c, 0xae, 0xae, 0xe4, 0x9d, 0x46, 0x7a, 0x09, - 0xc7, 0xca, 0xd9, 0xa3, 0x73, 0x32, 0x49, 0x8a, 0xe1, 0xec, 0xec, 0x5f, 0x18, 0x7b, 0xcd, 0xe3, - 0x18, 0x7b, 0x0d, 0xa3, 0xef, 0x4a, 0x18, 0xac, 0xf9, 0x10, 0xec, 0x22, 0x75, 0xbb, 0xec, 0xca, - 0xc6, 0xac, 0xdc, 0x72, 0x09, 0xb7, 0x47, 0xb6, 0x09, 0x5f, 0x46, 0x51, 0x25, 0x64, 0x1e, 0xd4, - 0x7d, 0xd9, 0x4d, 0x1d, 0xa6, 0xe8, 0xcb, 0xa6, 0xe4, 0xf3, 0xe6, 0x07, 0xf5, 0xc5, 0xf6, 0x9a, - 0x1f, 0xc2, 0x83, 0xc0, 0xeb, 0x45, 0xdb, 0xc6, 0x15, 0xae, 0x70, 0x37, 0xca, 0x7e, 0xc4, 0x46, - 0xb7, 0xdf, 0xaf, 0x9a, 0xe4, 0x17, 0x4d, 0xf2, 0x03, 0xc8, 0x3d, 0xfb, 0x18, 0x4e, 0x23, 0x76, - 0xa0, 0x5f, 0xc2, 0xf0, 0xb3, 0xd0, 0xa6, 0x3d, 0x48, 0x83, 0x8e, 0x20, 0x0d, 0xfe, 0x33, 0x48, - 0x57, 0x9e, 0x2c, 0x0a, 0xab, 0xc5, 0x86, 0xb4, 0xc7, 0xa6, 0xb6, 0xda, 0x5e, 0x44, 0x01, 0x23, - 0x8f, 0x12, 0x62, 0x43, 0x21, 0x5d, 0xe2, 0xd6, 0x5a, 0x91, 0x14, 0x03, 0xee, 0xce, 0x9f, 0xd2, - 0x13, 0x32, 0xee, 0xcd, 0xfe, 0xf4, 0xa0, 0xff, 0xd5, 0x02, 0xd1, 0xb7, 0x90, 0xda, 0xa8, 0xd1, - 0xd6, 0x60, 0x86, 0x7d, 0x1e, 0x9f, 0xb7, 0x5f, 0x06, 0xa7, 0x8e, 0xe8, 0x7b, 0xe8, 0xbb, 0xb7, - 0xa3, 0xed, 0x6f, 0x1d, 0x61, 0x2e, 0x3a, 0x6e, 0x77, 0x38, 0x1f, 0x21, 0xf3, 0xaf, 0x40, 0x3b, - 0xde, 0x2d, 0x22, 0x3d, 0xed, 0xba, 0xde, 0x41, 0xbd, 0x83, 0xd4, 0x7a, 0x41, 0x5b, 0x9d, 0xeb, - 0xd4, 0x55, 0xb7, 0x8f, 0x1d, 0x5d, 0x92, 0x9f, 0x99, 0xfb, 0x5f, 0x3d, 0xff, 0x1b, 0x00, 0x00, - 0xff, 0xff, 0xdd, 0xdb, 0x9c, 0x15, 0xd2, 0x04, 0x00, 0x00, +func init() { + proto.RegisterFile("store.proto", fileDescriptor_98bbca36ef968dfc) } -// Reference imports to suppress errors if they are not otherwise used. -var _ context.Context -var _ grpc.ClientConn - -// This is a compile-time assertion to ensure that this generated file -// is compatible with the grpc package it is being compiled against. -const _ = grpc.SupportPackageIsVersion4 - -// StoreClient is the client API for Store service. -// -// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://godoc.org/google.golang.org/grpc#ClientConn.NewStream. -type StoreClient interface { - Read(ctx context.Context, in *ReadRequest, opts ...grpc.CallOption) (*ReadResponse, error) - Write(ctx context.Context, in *WriteRequest, opts ...grpc.CallOption) (*WriteResponse, error) - Delete(ctx context.Context, in *DeleteRequest, opts ...grpc.CallOption) (*DeleteResponse, error) - List(ctx context.Context, in *ListRequest, opts ...grpc.CallOption) (Store_ListClient, error) -} - -type storeClient struct { - cc *grpc.ClientConn -} - -func NewStoreClient(cc *grpc.ClientConn) StoreClient { - return &storeClient{cc} -} - -func (c *storeClient) Read(ctx context.Context, in *ReadRequest, opts ...grpc.CallOption) (*ReadResponse, error) { - out := new(ReadResponse) - err := c.cc.Invoke(ctx, "/go.micro.store.Store/Read", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -func (c *storeClient) Write(ctx context.Context, in *WriteRequest, opts ...grpc.CallOption) (*WriteResponse, error) { - out := new(WriteResponse) - err := c.cc.Invoke(ctx, "/go.micro.store.Store/Write", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -func (c *storeClient) Delete(ctx context.Context, in *DeleteRequest, opts ...grpc.CallOption) (*DeleteResponse, error) { - out := new(DeleteResponse) - err := c.cc.Invoke(ctx, "/go.micro.store.Store/Delete", in, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - -func (c *storeClient) List(ctx context.Context, in *ListRequest, opts ...grpc.CallOption) (Store_ListClient, error) { - stream, err := c.cc.NewStream(ctx, &_Store_serviceDesc.Streams[0], "/go.micro.store.Store/List", opts...) - if err != nil { - return nil, err - } - x := &storeListClient{stream} - if err := x.ClientStream.SendMsg(in); err != nil { - return nil, err - } - if err := x.ClientStream.CloseSend(); err != nil { - return nil, err - } - return x, nil -} - -type Store_ListClient interface { - Recv() (*ListResponse, error) - grpc.ClientStream -} - -type storeListClient struct { - grpc.ClientStream -} - -func (x *storeListClient) Recv() (*ListResponse, error) { - m := new(ListResponse) - if err := x.ClientStream.RecvMsg(m); err != nil { - return nil, err - } - return m, nil -} - -// StoreServer is the server API for Store service. -type StoreServer interface { - Read(context.Context, *ReadRequest) (*ReadResponse, error) - Write(context.Context, *WriteRequest) (*WriteResponse, error) - Delete(context.Context, *DeleteRequest) (*DeleteResponse, error) - List(*ListRequest, Store_ListServer) error -} - -// UnimplementedStoreServer can be embedded to have forward compatible implementations. -type UnimplementedStoreServer struct { -} - -func (*UnimplementedStoreServer) Read(ctx context.Context, req *ReadRequest) (*ReadResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method Read not implemented") -} -func (*UnimplementedStoreServer) Write(ctx context.Context, req *WriteRequest) (*WriteResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method Write not implemented") -} -func (*UnimplementedStoreServer) Delete(ctx context.Context, req *DeleteRequest) (*DeleteResponse, error) { - return nil, status.Errorf(codes.Unimplemented, "method Delete not implemented") -} -func (*UnimplementedStoreServer) List(req *ListRequest, srv Store_ListServer) error { - return status.Errorf(codes.Unimplemented, "method List not implemented") -} - -func RegisterStoreServer(s *grpc.Server, srv StoreServer) { - s.RegisterService(&_Store_serviceDesc, srv) -} - -func _Store_Read_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(ReadRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(StoreServer).Read(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/go.micro.store.Store/Read", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(StoreServer).Read(ctx, req.(*ReadRequest)) - } - return interceptor(ctx, in, info, handler) -} - -func _Store_Write_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(WriteRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(StoreServer).Write(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/go.micro.store.Store/Write", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(StoreServer).Write(ctx, req.(*WriteRequest)) - } - return interceptor(ctx, in, info, handler) -} - -func _Store_Delete_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { - in := new(DeleteRequest) - if err := dec(in); err != nil { - return nil, err - } - if interceptor == nil { - return srv.(StoreServer).Delete(ctx, in) - } - info := &grpc.UnaryServerInfo{ - Server: srv, - FullMethod: "/go.micro.store.Store/Delete", - } - handler := func(ctx context.Context, req interface{}) (interface{}, error) { - return srv.(StoreServer).Delete(ctx, req.(*DeleteRequest)) - } - return interceptor(ctx, in, info, handler) -} - -func _Store_List_Handler(srv interface{}, stream grpc.ServerStream) error { - m := new(ListRequest) - if err := stream.RecvMsg(m); err != nil { - return err - } - return srv.(StoreServer).List(m, &storeListServer{stream}) -} - -type Store_ListServer interface { - Send(*ListResponse) error - grpc.ServerStream -} - -type storeListServer struct { - grpc.ServerStream -} - -func (x *storeListServer) Send(m *ListResponse) error { - return x.ServerStream.SendMsg(m) -} - -var _Store_serviceDesc = grpc.ServiceDesc{ - ServiceName: "go.micro.store.Store", - HandlerType: (*StoreServer)(nil), - Methods: []grpc.MethodDesc{ - { - MethodName: "Read", - Handler: _Store_Read_Handler, - }, - { - MethodName: "Write", - Handler: _Store_Write_Handler, - }, - { - MethodName: "Delete", - Handler: _Store_Delete_Handler, - }, - }, - Streams: []grpc.StreamDesc{ - { - StreamName: "List", - Handler: _Store_List_Handler, - ServerStreams: true, - }, - }, - Metadata: "store/service/proto/store.proto", +var fileDescriptor_98bbca36ef968dfc = []byte{ + // 552 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x54, 0x5d, 0x8b, 0xd3, 0x40, + 0x14, 0x6d, 0x9a, 0x34, 0xdb, 0xdc, 0x76, 0x6b, 0x1d, 0xa4, 0x94, 0xda, 0x95, 0x38, 0x4f, 0x01, + 0x21, 0xac, 0x15, 0x3f, 0x1e, 0x05, 0xab, 0xa8, 0x08, 0xc2, 0x28, 0x0a, 0xbe, 0xa5, 0xdb, 0xa9, + 0x84, 0x66, 0x77, 0x62, 0x66, 0xba, 0x6c, 0x7f, 0xa0, 0xff, 0x4b, 0xe6, 0x2b, 0x4d, 0xd3, 0xc4, + 0x87, 0x7d, 0x9b, 0x7b, 0xe7, 0xce, 0x39, 0xf7, 0xdc, 0x7b, 0x12, 0x18, 0x70, 0xc1, 0x0a, 0x1a, + 0xe7, 0x05, 0x13, 0x0c, 0x8d, 0x7e, 0xb3, 0xf8, 0x3a, 0xbd, 0x2a, 0x58, 0xac, 0xb2, 0xf8, 0x23, + 0xf8, 0x84, 0x5e, 0xb1, 0x62, 0x8d, 0xc6, 0xe0, 0x6e, 0xe9, 0x7e, 0xea, 0x84, 0x4e, 0x14, 0x10, + 0x79, 0x44, 0x8f, 0xa0, 0x77, 0x9b, 0x64, 0x3b, 0x3a, 0xed, 0x86, 0x4e, 0x34, 0x24, 0x3a, 0x40, + 0x13, 0xf0, 0xe9, 0x5d, 0x9e, 0x16, 0xfb, 0xa9, 0x1b, 0x3a, 0x91, 0x4b, 0x4c, 0x84, 0xb7, 0x30, + 0x20, 0x34, 0x59, 0x7f, 0xcd, 0x45, 0xca, 0x6e, 0xb8, 0x2c, 0xcb, 0x0b, 0xba, 0x49, 0xef, 0x14, + 0x62, 0x9f, 0x98, 0x48, 0xe6, 0xf9, 0x6e, 0x23, 0xf3, 0x5d, 0x9d, 0xd7, 0x91, 0x24, 0xcb, 0xd2, + 0xeb, 0x54, 0x28, 0x54, 0x8f, 0xe8, 0x40, 0x56, 0xb3, 0xcd, 0x86, 0x53, 0x31, 0xf5, 0x54, 0xda, + 0x44, 0xf8, 0x87, 0x26, 0x23, 0xf4, 0xcf, 0x8e, 0x72, 0xd1, 0xd0, 0xfb, 0x4b, 0x38, 0x63, 0xba, + 0x13, 0xc5, 0x33, 0x58, 0x3c, 0x8e, 0x8f, 0x95, 0xc7, 0x95, 0x66, 0x89, 0xad, 0xc5, 0x6f, 0x61, + 0xa8, 0x71, 0x79, 0xce, 0x6e, 0x38, 0x45, 0x97, 0x70, 0x56, 0xa8, 0xf1, 0xf0, 0xa9, 0x13, 0xba, + 0xd1, 0x60, 0x31, 0x39, 0x85, 0x91, 0xd7, 0xc4, 0x96, 0xe1, 0x37, 0x30, 0xfc, 0x59, 0xa4, 0x82, + 0x56, 0xe6, 0x60, 0xc6, 0xe5, 0x54, 0xc7, 0x25, 0x5b, 0x16, 0x22, 0x53, 0xcd, 0xb9, 0x44, 0x1e, + 0xf1, 0xad, 0x79, 0x69, 0x45, 0xc5, 0xe0, 0x6b, 0x50, 0xf5, 0xb2, 0x9d, 0xda, 0x54, 0xa1, 0x57, + 0x75, 0xc9, 0xf3, 0xfa, 0x83, 0x6a, 0x63, 0x07, 0xcd, 0x0f, 0xe0, 0xdc, 0xf0, 0x6a, 0xd1, 0x32, + 0xb1, 0xa4, 0x19, 0x2d, 0x4b, 0xf1, 0x2f, 0x9b, 0x68, 0x9f, 0xf7, 0xeb, 0x3a, 0xf9, 0x45, 0x9d, + 0xfc, 0x08, 0xf2, 0xc0, 0x3e, 0x86, 0x91, 0xc5, 0x36, 0xf4, 0x5b, 0x18, 0x7c, 0x49, 0xb9, 0x68, + 0x36, 0x52, 0xd0, 0x62, 0xa4, 0xe0, 0x9e, 0x46, 0x5a, 0x6a, 0x32, 0x2b, 0xac, 0x62, 0x1b, 0xa7, + 0xd9, 0x36, 0x95, 0xd6, 0x0e, 0x22, 0x22, 0x18, 0x6a, 0x14, 0x63, 0x1b, 0x04, 0xde, 0x96, 0xee, + 0xe5, 0x28, 0xdc, 0x28, 0x20, 0xea, 0xfc, 0xd9, 0xeb, 0x3b, 0xe3, 0x2e, 0x46, 0x30, 0x5e, 0x26, + 0x22, 0x59, 0x25, 0x9c, 0x72, 0x43, 0x8a, 0x9f, 0xc3, 0xc3, 0x4a, 0xce, 0x40, 0xcc, 0x21, 0x58, + 0xdb, 0xa4, 0xf2, 0x5e, 0x40, 0x0e, 0x09, 0xfc, 0x0c, 0xce, 0xbf, 0x27, 0xab, 0xac, 0xc4, 0x40, + 0x33, 0xe8, 0xdb, 0x5b, 0x33, 0xa7, 0x32, 0xc6, 0x11, 0x8c, 0x6c, 0xb1, 0x01, 0x9f, 0x80, 0x2f, + 0x54, 0xc6, 0x20, 0x9b, 0x68, 0xf1, 0xd7, 0x85, 0xde, 0x37, 0x29, 0x13, 0xbd, 0x03, 0x4f, 0x7e, + 0x08, 0xa8, 0xf1, 0xb3, 0x31, 0xa4, 0xb3, 0x79, 0xf3, 0xa5, 0xd9, 0x63, 0x07, 0x7d, 0x80, 0x9e, + 0x72, 0x16, 0x6a, 0x76, 0xa2, 0x85, 0xb9, 0x68, 0xb9, 0x2d, 0x71, 0x3e, 0x81, 0xaf, 0x3d, 0x82, + 0x5a, 0x5c, 0x65, 0x91, 0x9e, 0xb4, 0x5d, 0x97, 0x50, 0xef, 0xc1, 0x93, 0x9b, 0x42, 0x8d, 0x7b, + 0x6d, 0xd5, 0x55, 0x5d, 0x2e, 0xee, 0x5c, 0x3a, 0x88, 0x40, 0x50, 0xae, 0x0c, 0x85, 0x27, 0xac, + 0xb5, 0x0d, 0xcf, 0x9e, 0xfe, 0xa7, 0xa2, 0xaa, 0x52, 0xaf, 0xe9, 0x54, 0xe5, 0xd1, 0xae, 0x4f, + 0x55, 0x1e, 0x6f, 0x17, 0x77, 0x56, 0xbe, 0xfa, 0xd9, 0xbf, 0xf8, 0x17, 0x00, 0x00, 0xff, 0xff, + 0x18, 0xa5, 0x9b, 0x82, 0xfb, 0x05, 0x00, 0x00, } diff --git a/store/service/proto/store.pb.micro.go b/store/service/proto/store.pb.micro.go index 4cdba07f..2c5ca29f 100644 --- a/store/service/proto/store.pb.micro.go +++ b/store/service/proto/store.pb.micro.go @@ -1,5 +1,5 @@ // Code generated by protoc-gen-micro. DO NOT EDIT. -// source: store/service/proto/store.proto +// source: store.proto package go_micro_store @@ -38,6 +38,8 @@ type StoreService interface { Write(ctx context.Context, in *WriteRequest, opts ...client.CallOption) (*WriteResponse, error) Delete(ctx context.Context, in *DeleteRequest, opts ...client.CallOption) (*DeleteResponse, error) List(ctx context.Context, in *ListRequest, opts ...client.CallOption) (Store_ListService, error) + Databases(ctx context.Context, in *DatabasesRequest, opts ...client.CallOption) (*DatabasesResponse, error) + Tables(ctx context.Context, in *TablesRequest, opts ...client.CallOption) (*TablesResponse, error) } type storeService struct { @@ -131,6 +133,26 @@ func (x *storeServiceList) Recv() (*ListResponse, error) { return m, nil } +func (c *storeService) Databases(ctx context.Context, in *DatabasesRequest, opts ...client.CallOption) (*DatabasesResponse, error) { + req := c.c.NewRequest(c.name, "Store.Databases", in) + out := new(DatabasesResponse) + err := c.c.Call(ctx, req, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + +func (c *storeService) Tables(ctx context.Context, in *TablesRequest, opts ...client.CallOption) (*TablesResponse, error) { + req := c.c.NewRequest(c.name, "Store.Tables", in) + out := new(TablesResponse) + err := c.c.Call(ctx, req, out, opts...) + if err != nil { + return nil, err + } + return out, nil +} + // Server API for Store service type StoreHandler interface { @@ -138,6 +160,8 @@ type StoreHandler interface { Write(context.Context, *WriteRequest, *WriteResponse) error Delete(context.Context, *DeleteRequest, *DeleteResponse) error List(context.Context, *ListRequest, Store_ListStream) error + Databases(context.Context, *DatabasesRequest, *DatabasesResponse) error + Tables(context.Context, *TablesRequest, *TablesResponse) error } func RegisterStoreHandler(s server.Server, hdlr StoreHandler, opts ...server.HandlerOption) error { @@ -146,6 +170,8 @@ func RegisterStoreHandler(s server.Server, hdlr StoreHandler, opts ...server.Han Write(ctx context.Context, in *WriteRequest, out *WriteResponse) error Delete(ctx context.Context, in *DeleteRequest, out *DeleteResponse) error List(ctx context.Context, stream server.Stream) error + Databases(ctx context.Context, in *DatabasesRequest, out *DatabasesResponse) error + Tables(ctx context.Context, in *TablesRequest, out *TablesResponse) error } type Store struct { store @@ -209,3 +235,11 @@ func (x *storeListStream) RecvMsg(m interface{}) error { func (x *storeListStream) Send(m *ListResponse) error { return x.stream.Send(m) } + +func (h *storeHandler) Databases(ctx context.Context, in *DatabasesRequest, out *DatabasesResponse) error { + return h.StoreHandler.Databases(ctx, in, out) +} + +func (h *storeHandler) Tables(ctx context.Context, in *TablesRequest, out *TablesResponse) error { + return h.StoreHandler.Tables(ctx, in, out) +} diff --git a/store/service/proto/store.proto b/store/service/proto/store.proto index a6462ad2..460cf049 100644 --- a/store/service/proto/store.proto +++ b/store/service/proto/store.proto @@ -7,6 +7,8 @@ service Store { rpc Write(WriteRequest) returns (WriteResponse) {}; rpc Delete(DeleteRequest) returns (DeleteResponse) {}; rpc List(ListRequest) returns (stream ListResponse) {}; + rpc Databases(DatabasesRequest) returns (DatabasesResponse) {}; + rpc Tables(TablesRequest) returns (TablesResponse) {}; } message Record { @@ -72,3 +74,17 @@ message ListResponse { reserved 1; //repeated Record records = 1; repeated string keys = 2; } + +message DatabasesRequest {} + +message DatabasesResponse { + repeated string databases = 1; +} + +message TablesRequest { + string database = 1; +} + +message TablesResponse { + repeated string tables = 1; +} From 53549b6b3089fc23345262036510300fd1e3b141 Mon Sep 17 00:00:00 2001 From: Asim Aslam Date: Thu, 9 Apr 2020 17:56:13 +0100 Subject: [PATCH 04/44] Add options for Database/Table (#1516) * Add options for Database/Table * fix opts --- store/options.go | 40 +++++++++++++++++++++++++++++++++++++++- 1 file changed, 39 insertions(+), 1 deletion(-) diff --git a/store/options.go b/store/options.go index 01d60c80..27224731 100644 --- a/store/options.go +++ b/store/options.go @@ -54,6 +54,7 @@ func WithContext(c context.Context) Option { // ReadOptions configures an individual Read operation type ReadOptions struct { + Database, Table string // Prefix returns all records that are prefixed with key Prefix bool // Suffix returns all records that have the suffix key @@ -67,6 +68,14 @@ type ReadOptions struct { // ReadOption sets values in ReadOptions type ReadOption func(r *ReadOptions) +// ReadFrom the database and table +func ReadFrom(database, table string) ReadOption { + return func(r *ReadOptions) { + r.Database = database + r.Table = table + } +} + // ReadPrefix returns all records that are prefixed with key func ReadPrefix() ReadOption { return func(r *ReadOptions) { @@ -98,6 +107,7 @@ func ReadOffset(o uint) ReadOption { // WriteOptions configures an individual Write operation // If Expiry and TTL are set TTL takes precedence type WriteOptions struct { + Database, Table string // Expiry is the time the record expires Expiry time.Time // TTL is the time until the record expires @@ -107,6 +117,14 @@ type WriteOptions struct { // WriteOption sets values in WriteOptions type WriteOption func(w *WriteOptions) +// WriteTo the database and table +func WriteTo(database, table string) WriteOption { + return func(w *WriteOptions) { + w.Database = database + w.Table = table + } +} + // WriteExpiry is the time the record expires func WriteExpiry(t time.Time) WriteOption { return func(w *WriteOptions) { @@ -122,13 +140,25 @@ func WriteTTL(d time.Duration) WriteOption { } // DeleteOptions configures an individual Delete operation -type DeleteOptions struct{} +type DeleteOptions struct { + Database, Table string +} // DeleteOption sets values in DeleteOptions type DeleteOption func(d *DeleteOptions) +// DeleteFrom the database and table +func DeleteFrom(database, table string) DeleteOption { + return func(d *DeleteOptions) { + d.Database = database + d.Table = table + } +} + // ListOptions configures an individual List operation type ListOptions struct { + // List from the following + Database, Table string // Prefix returns all keys that are prefixed with key Prefix string // Suffix returns all keys that end with key @@ -142,6 +172,14 @@ type ListOptions struct { // ListOption sets values in ListOptions type ListOption func(l *ListOptions) +// ListFrom the database and table +func ListFrom(database, table string) ListOption { + return func(l *ListOptions) { + l.Database = database + l.Table = table + } +} + // ListPrefix returns all keys that are prefixed with key func ListPrefix(p string) ListOption { return func(l *ListOptions) { From 6a666c9c7db871c0b88152ad5a84e9f4a92a845c Mon Sep 17 00:00:00 2001 From: Jake Sanders Date: Thu, 9 Apr 2020 19:38:43 +0100 Subject: [PATCH 05/44] Add json tags to store.Record (#1518) --- store/store.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/store/store.go b/store/store.go index d073f4e1..c949f939 100644 --- a/store/store.go +++ b/store/store.go @@ -36,7 +36,7 @@ type Store interface { // Record is an item stored or retrieved from a Store type Record struct { - Key string - Value []byte - Expiry time.Duration + Key string `json:"key"` + Value []byte `json:"value"` + Expiry time.Duration `json:"expiry,omitempty"` } From 9a685b2df5ef441fd3d0fd4dadd2b6bb6c9d3393 Mon Sep 17 00:00:00 2001 From: Asim Aslam Date: Fri, 10 Apr 2020 17:15:20 +0100 Subject: [PATCH 06/44] delete k8s registry (#1522) --- config/cmd/cmd.go | 10 +- registry/kubernetes/README.md | 66 ------- registry/kubernetes/kubernetes.go | 289 ------------------------------ registry/kubernetes/watcher.go | 263 --------------------------- 4 files changed, 4 insertions(+), 624 deletions(-) delete mode 100644 registry/kubernetes/README.md delete mode 100644 registry/kubernetes/kubernetes.go delete mode 100644 registry/kubernetes/watcher.go diff --git a/config/cmd/cmd.go b/config/cmd/cmd.go index 674affba..44071b71 100644 --- a/config/cmd/cmd.go +++ b/config/cmd/cmd.go @@ -42,7 +42,6 @@ import ( // registries "github.com/micro/go-micro/v2/registry/etcd" - kreg "github.com/micro/go-micro/v2/registry/kubernetes" "github.com/micro/go-micro/v2/registry/mdns" rmem "github.com/micro/go-micro/v2/registry/memory" regSrv "github.com/micro/go-micro/v2/registry/service" @@ -328,11 +327,10 @@ var ( } DefaultRegistries = map[string]func(...registry.Option) registry.Registry{ - "service": regSrv.NewRegistry, - "etcd": etcd.NewRegistry, - "mdns": mdns.NewRegistry, - "memory": rmem.NewRegistry, - "kubernetes": kreg.NewRegistry, + "service": regSrv.NewRegistry, + "etcd": etcd.NewRegistry, + "mdns": mdns.NewRegistry, + "memory": rmem.NewRegistry, } DefaultSelectors = map[string]func(...selector.Option) selector.Selector{ diff --git a/registry/kubernetes/README.md b/registry/kubernetes/README.md deleted file mode 100644 index 80282db4..00000000 --- a/registry/kubernetes/README.md +++ /dev/null @@ -1,66 +0,0 @@ -# Kubernetes Registry Plugin for micro -This is a plugin for go-micro that allows you to use Kubernetes as a registry. - - -## Overview -This registry plugin makes use of Annotations and Labels on a Kubernetes pod -to build a service discovery mechanism. - - -## RBAC -If your Kubernetes cluster has RBAC enabled, a role and role binding -will need to be created to allow this plugin to `list` and `patch` pods. - -A cluster role can be used to specify the `list` and `patch` -requirements, while a role binding per namespace can be used to apply -the cluster role. The example RBAC configs below assume your Micro-based -services are running in the `test` namespace, and the pods that contain -the services are using the `micro-services` service account. - -``` -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: micro-registry -rules: -- apiGroups: - - "" - resources: - - pods - verbs: - - list - - patch - - watch -``` - -``` -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: micro-registry -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: micro-registry -subjects: -- kind: ServiceAccount - name: micro-services - namespace: test -``` - - -## Gotchas -* Registering/Deregistering relies on the HOSTNAME Environment Variable, which inside a pod -is the place where it can be retrieved from. (This needs improving) - - -## Connecting to the Kubernetes API -### Within a pod -If the `--registry_address` flag is omitted, the plugin will securely connect to -the Kubernetes API using the pods "Service Account". No extra configuration is necessary. - -Find out more about service accounts here. http://kubernetes.io/docs/user-guide/accessing-the-cluster/ - -### Outside of Kubernetes -Some functions of the plugin should work, but its not been heavily tested. -Currently no TLS support. diff --git a/registry/kubernetes/kubernetes.go b/registry/kubernetes/kubernetes.go deleted file mode 100644 index 81719376..00000000 --- a/registry/kubernetes/kubernetes.go +++ /dev/null @@ -1,289 +0,0 @@ -// Package kubernetes provides a kubernetes registry -package kubernetes - -import ( - "encoding/json" - "errors" - "fmt" - "os" - "regexp" - "strings" - "time" - - "github.com/micro/go-micro/v2/registry" - "github.com/micro/go-micro/v2/util/kubernetes/client" -) - -type kregistry struct { - client client.Client - timeout time.Duration - options registry.Options -} - -var ( - // used on pods as labels & services to select - // eg: svcSelectorPrefix+"svc.name" - servicePrefix = "go.micro/" - serviceValue = "service" - - labelTypeKey = "micro" - labelTypeValue = "service" - - // used on k8s services to scope a serialised - // micro service by pod name - annotationPrefix = "go.micro/" - - // Pod status - podRunning = "Running" - - // label name regex - labelRe = regexp.MustCompilePOSIX("[-A-Za-z0-9_.]") -) - -// podSelector -var podSelector = map[string]string{ - labelTypeKey: labelTypeValue, -} - -func configure(k *kregistry, opts ...registry.Option) error { - for _, o := range opts { - o(&k.options) - } - - // get first host - var host string - if len(k.options.Addrs) > 0 && len(k.options.Addrs[0]) > 0 { - host = k.options.Addrs[0] - } - - if k.options.Timeout == 0 { - k.options.Timeout = time.Second * 1 - } - - // if no hosts setup, assume InCluster - var c client.Client - - if len(host) > 0 { - c = client.NewLocalClient(host) - } else { - c = client.NewClusterClient() - } - - k.client = c - k.timeout = k.options.Timeout - - return nil -} - -// serviceName generates a valid service name for k8s labels -func serviceName(name string) string { - aname := make([]byte, len(name)) - - for i, r := range []byte(name) { - if !labelRe.Match([]byte{r}) { - aname[i] = '_' - continue - } - aname[i] = r - } - - return string(aname) -} - -// Init allows reconfig of options -func (c *kregistry) Init(opts ...registry.Option) error { - return configure(c, opts...) -} - -// Options returns the registry Options -func (c *kregistry) Options() registry.Options { - return c.options -} - -// Register sets a service selector label and an annotation with a -// serialised version of the service passed in. -func (c *kregistry) Register(s *registry.Service, opts ...registry.RegisterOption) error { - if len(s.Nodes) == 0 { - return errors.New("no nodes") - } - - // TODO: grab podname from somewhere better than this. - podName := os.Getenv("HOSTNAME") - svcName := s.Name - - // encode micro service - b, err := json.Marshal(s) - if err != nil { - return err - } - /// marshalled service - svc := string(b) - - pod := &client.Pod{ - Metadata: &client.Metadata{ - Labels: map[string]string{ - // micro: service - labelTypeKey: labelTypeValue, - // micro/service/name: service - servicePrefix + serviceName(svcName): serviceValue, - }, - Annotations: map[string]string{ - // micro/service/name: definition - annotationPrefix + serviceName(svcName): svc, - }, - }, - } - - return c.client.Update(&client.Resource{ - Name: podName, - Kind: "pod", - Value: pod, - }) -} - -// Deregister nils out any things set in Register -func (c *kregistry) Deregister(s *registry.Service) error { - if len(s.Nodes) == 0 { - return errors.New("you must deregister at least one node") - } - - // TODO: grab podname from somewhere better than this. - podName := os.Getenv("HOSTNAME") - svcName := s.Name - - pod := &client.Pod{ - Metadata: &client.Metadata{ - Labels: map[string]string{ - servicePrefix + serviceName(svcName): "", - }, - Annotations: map[string]string{ - annotationPrefix + serviceName(svcName): "", - }, - }, - } - - return c.client.Update(&client.Resource{ - Name: podName, - Kind: "pod", - Value: pod, - }) -} - -// GetService will get all the pods with the given service selector, -// and build services from the annotations. -func (c *kregistry) GetService(name string) ([]*registry.Service, error) { - var pods client.PodList - - if err := c.client.Get(&client.Resource{ - Kind: "pod", - Value: &pods, - }, map[string]string{ - servicePrefix + serviceName(name): serviceValue, - }); err != nil { - return nil, err - } - - if len(pods.Items) == 0 { - return nil, registry.ErrNotFound - } - - // svcs mapped by version - svcs := make(map[string]*registry.Service) - - // loop through items - for _, pod := range pods.Items { - if pod.Status.Phase != podRunning { - continue - } - - // get serialised service from annotation - svcStr, ok := pod.Metadata.Annotations[annotationPrefix+serviceName(name)] - if !ok { - continue - } - - // unmarshal service string - var svc registry.Service - - if err := json.Unmarshal([]byte(svcStr), &svc); err != nil { - return nil, fmt.Errorf("could not unmarshal service '%s' from pod annotation", name) - } - - // merge up pod service & ip with versioned service. - vs, ok := svcs[svc.Version] - if !ok { - svcs[svc.Version] = &svc - continue - } - - vs.Nodes = append(vs.Nodes, svc.Nodes...) - } - - list := make([]*registry.Service, 0, len(svcs)) - for _, val := range svcs { - list = append(list, val) - } - return list, nil -} - -// ListServices will list all the service names -func (c *kregistry) ListServices() ([]*registry.Service, error) { - var pods client.PodList - - if err := c.client.Get(&client.Resource{ - Kind: "pod", - Value: &pods, - }, podSelector); err != nil { - return nil, err - } - - // svcs mapped by name - svcs := make(map[string]bool) - - for _, pod := range pods.Items { - if pod.Status.Phase != podRunning { - continue - } - for k, v := range pod.Metadata.Annotations { - if !strings.HasPrefix(k, annotationPrefix) { - continue - } - - // we have to unmarshal the annotation itself since the - // key is encoded to match the regex restriction. - var svc registry.Service - - if err := json.Unmarshal([]byte(v), &svc); err != nil { - continue - } - - svcs[svc.Name] = true - } - } - - var list []*registry.Service - - for val := range svcs { - list = append(list, ®istry.Service{Name: val}) - } - - return list, nil -} - -// Watch returns a kubernetes watcher -func (c *kregistry) Watch(opts ...registry.WatchOption) (registry.Watcher, error) { - return newWatcher(c, opts...) -} - -func (c *kregistry) String() string { - return "kubernetes" -} - -// NewRegistry creates a kubernetes registry -func NewRegistry(opts ...registry.Option) registry.Registry { - k := &kregistry{ - options: registry.Options{}, - } - configure(k, opts...) - return k -} diff --git a/registry/kubernetes/watcher.go b/registry/kubernetes/watcher.go deleted file mode 100644 index 89bffed6..00000000 --- a/registry/kubernetes/watcher.go +++ /dev/null @@ -1,263 +0,0 @@ -package kubernetes - -import ( - "encoding/json" - "errors" - "strings" - "sync" - - "github.com/micro/go-micro/v2/logger" - "github.com/micro/go-micro/v2/registry" - "github.com/micro/go-micro/v2/util/kubernetes/client" -) - -type k8sWatcher struct { - registry *kregistry - watcher client.Watcher - next chan *registry.Result - stop chan bool - - sync.RWMutex - pods map[string]*client.Pod -} - -// build a cache of pods when the watcher starts. -func (k *k8sWatcher) updateCache() ([]*registry.Result, error) { - var pods client.PodList - - if err := k.registry.client.Get(&client.Resource{ - Kind: "pod", - Value: &pods, - }, podSelector); err != nil { - return nil, err - } - - var results []*registry.Result - - for _, pod := range pods.Items { - rslts := k.buildPodResults(&pod, nil) - - for _, r := range rslts { - results = append(results, r) - } - - k.Lock() - k.pods[pod.Metadata.Name] = &pod - k.Unlock() - } - - return results, nil -} - -// look through pod annotations, compare against cache if present -// and return a list of results to send down the wire. -func (k *k8sWatcher) buildPodResults(pod *client.Pod, cache *client.Pod) []*registry.Result { - var results []*registry.Result - ignore := make(map[string]bool) - - if pod.Metadata != nil { - for ak, av := range pod.Metadata.Annotations { - // check this annotation kv is a service notation - if !strings.HasPrefix(ak, annotationPrefix) { - continue - } - - if len(av) == 0 { - continue - } - - // ignore when we check the cached annotations - // as we take care of it here - ignore[ak] = true - - // compare aginst cache. - var cacheExists bool - var cav string - - if cache != nil && cache.Metadata != nil { - cav, cacheExists = cache.Metadata.Annotations[ak] - if cacheExists && len(cav) > 0 && cav == av { - // service notation exists and is identical - - // no change result required. - continue - } - } - - rslt := ®istry.Result{} - if cacheExists { - rslt.Action = "update" - } else { - rslt.Action = "create" - } - - // unmarshal service notation from annotation value - err := json.Unmarshal([]byte(av), &rslt.Service) - if err != nil { - continue - } - - results = append(results, rslt) - } - } - - // loop through cache annotations to find services - // not accounted for above, and "delete" them. - if cache != nil && cache.Metadata != nil { - for ak, av := range cache.Metadata.Annotations { - if ignore[ak] { - continue - } - - // check this annotation kv is a service notation - if !strings.HasPrefix(ak, annotationPrefix) { - continue - } - - rslt := ®istry.Result{Action: "delete"} - // unmarshal service notation from annotation value - err := json.Unmarshal([]byte(av), &rslt.Service) - if err != nil { - continue - } - - results = append(results, rslt) - } - } - - return results -} - -// handleEvent will taken an event from the k8s pods API and do the correct -// things with the result, based on the local cache. -func (k *k8sWatcher) handleEvent(event client.Event) { - var pod client.Pod - if err := json.Unmarshal([]byte(event.Object), &pod); err != nil { - if logger.V(logger.InfoLevel, logger.DefaultLogger) { - logger.Info("K8s Watcher: Couldnt unmarshal event object from pod") - } - return - } - - switch event.Type { - case client.Modified: - // Pod was modified - - k.RLock() - cache := k.pods[pod.Metadata.Name] - k.RUnlock() - - // service could have been added, edited or removed. - var results []*registry.Result - - if pod.Status.Phase == podRunning { - results = k.buildPodResults(&pod, cache) - } else { - // passing in cache might not return all results - results = k.buildPodResults(&pod, nil) - } - - for _, result := range results { - // pod isnt running - if pod.Status.Phase != podRunning { - result.Action = "delete" - } - - select { - case k.next <- result: - case <-k.stop: - return - } - } - - k.Lock() - k.pods[pod.Metadata.Name] = &pod - k.Unlock() - return - - case client.Deleted: - // Pod was deleted - // passing in cache might not return all results - results := k.buildPodResults(&pod, nil) - - for _, result := range results { - result.Action = "delete" - select { - case k.next <- result: - case <-k.stop: - return - } - } - - k.Lock() - delete(k.pods, pod.Metadata.Name) - k.Unlock() - return - } - -} - -// Next will block until a new result comes in -func (k *k8sWatcher) Next() (*registry.Result, error) { - select { - case r := <-k.next: - return r, nil - case <-k.stop: - return nil, errors.New("watcher stopped") - } -} - -// Stop will cancel any requests, and close channels -func (k *k8sWatcher) Stop() { - select { - case <-k.stop: - return - default: - k.watcher.Stop() - close(k.stop) - } -} - -func newWatcher(kr *kregistry, opts ...registry.WatchOption) (registry.Watcher, error) { - var wo registry.WatchOptions - for _, o := range opts { - o(&wo) - } - - selector := podSelector - if len(wo.Service) > 0 { - selector = map[string]string{ - servicePrefix + serviceName(wo.Service): serviceValue, - } - } - - // Create watch request - watcher, err := kr.client.Watch(&client.Resource{ - Kind: "pod", - }, client.WatchParams(selector)) - if err != nil { - return nil, err - } - - k := &k8sWatcher{ - registry: kr, - watcher: watcher, - next: make(chan *registry.Result), - stop: make(chan bool), - pods: make(map[string]*client.Pod), - } - - // update cache, but dont emit changes - if _, err := k.updateCache(); err != nil { - return nil, err - } - - // range over watch request changes, and invoke - // the update event - go func() { - for event := range watcher.Chan() { - k.handleEvent(event) - } - }() - - return k, nil -} From d134b469be6a2e8a060e40dcfa4a0b83a485fa58 Mon Sep 17 00:00:00 2001 From: Asim Aslam Date: Fri, 10 Apr 2020 17:17:24 +0100 Subject: [PATCH 07/44] rename file --- registry/{watcher_test.go => mdns_watcher_test.go} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename registry/{watcher_test.go => mdns_watcher_test.go} (100%) diff --git a/registry/watcher_test.go b/registry/mdns_watcher_test.go similarity index 100% rename from registry/watcher_test.go rename to registry/mdns_watcher_test.go From 4fd12430d0f6c280bd1ed1233e6d4975983dbbbe Mon Sep 17 00:00:00 2001 From: Asim Aslam Date: Fri, 10 Apr 2020 17:19:26 +0100 Subject: [PATCH 08/44] cleanup mdns files --- registry/encoding.go | 73 --------------------------------------- registry/encoding_test.go | 65 ---------------------------------- registry/mdns_registry.go | 67 +++++++++++++++++++++++++++++++++++ registry/mdns_test.go | 60 ++++++++++++++++++++++++++++++++ 4 files changed, 127 insertions(+), 138 deletions(-) delete mode 100644 registry/encoding.go delete mode 100644 registry/encoding_test.go diff --git a/registry/encoding.go b/registry/encoding.go deleted file mode 100644 index 750f88f8..00000000 --- a/registry/encoding.go +++ /dev/null @@ -1,73 +0,0 @@ -package registry - -import ( - "bytes" - "compress/zlib" - "encoding/hex" - "encoding/json" - "io/ioutil" - "strings" -) - -func encode(txt *mdnsTxt) ([]string, error) { - b, err := json.Marshal(txt) - if err != nil { - return nil, err - } - - var buf bytes.Buffer - defer buf.Reset() - - w := zlib.NewWriter(&buf) - if _, err := w.Write(b); err != nil { - return nil, err - } - w.Close() - - encoded := hex.EncodeToString(buf.Bytes()) - - // individual txt limit - if len(encoded) <= 255 { - return []string{encoded}, nil - } - - // split encoded string - var record []string - - for len(encoded) > 255 { - record = append(record, encoded[:255]) - encoded = encoded[255:] - } - - record = append(record, encoded) - - return record, nil -} - -func decode(record []string) (*mdnsTxt, error) { - encoded := strings.Join(record, "") - - hr, err := hex.DecodeString(encoded) - if err != nil { - return nil, err - } - - br := bytes.NewReader(hr) - zr, err := zlib.NewReader(br) - if err != nil { - return nil, err - } - - rbuf, err := ioutil.ReadAll(zr) - if err != nil { - return nil, err - } - - var txt *mdnsTxt - - if err := json.Unmarshal(rbuf, &txt); err != nil { - return nil, err - } - - return txt, nil -} diff --git a/registry/encoding_test.go b/registry/encoding_test.go deleted file mode 100644 index ff9ae97e..00000000 --- a/registry/encoding_test.go +++ /dev/null @@ -1,65 +0,0 @@ -package registry - -import ( - "testing" -) - -func TestEncoding(t *testing.T) { - testData := []*mdnsTxt{ - { - Version: "1.0.0", - Metadata: map[string]string{ - "foo": "bar", - }, - Endpoints: []*Endpoint{ - { - Name: "endpoint1", - Request: &Value{ - Name: "request", - Type: "request", - }, - Response: &Value{ - Name: "response", - Type: "response", - }, - Metadata: map[string]string{ - "foo1": "bar1", - }, - }, - }, - }, - } - - for _, d := range testData { - encoded, err := encode(d) - if err != nil { - t.Fatal(err) - } - - for _, txt := range encoded { - if len(txt) > 255 { - t.Fatalf("One of parts for txt is %d characters", len(txt)) - } - } - - decoded, err := decode(encoded) - if err != nil { - t.Fatal(err) - } - - if decoded.Version != d.Version { - t.Fatalf("Expected version %s got %s", d.Version, decoded.Version) - } - - if len(decoded.Endpoints) != len(d.Endpoints) { - t.Fatalf("Expected %d endpoints, got %d", len(d.Endpoints), len(decoded.Endpoints)) - } - - for k, v := range d.Metadata { - if val := decoded.Metadata[k]; val != v { - t.Fatalf("Expected %s=%s got %s=%s", k, v, k, val) - } - } - } - -} diff --git a/registry/mdns_registry.go b/registry/mdns_registry.go index 9dbca6de..7e614840 100644 --- a/registry/mdns_registry.go +++ b/registry/mdns_registry.go @@ -2,8 +2,13 @@ package registry import ( + "bytes" + "compress/zlib" "context" + "encoding/hex" + "encoding/json" "fmt" + "io/ioutil" "net" "strconv" "strings" @@ -49,6 +54,68 @@ type mdnsRegistry struct { listener chan *mdns.ServiceEntry } +func encode(txt *mdnsTxt) ([]string, error) { + b, err := json.Marshal(txt) + if err != nil { + return nil, err + } + + var buf bytes.Buffer + defer buf.Reset() + + w := zlib.NewWriter(&buf) + if _, err := w.Write(b); err != nil { + return nil, err + } + w.Close() + + encoded := hex.EncodeToString(buf.Bytes()) + + // individual txt limit + if len(encoded) <= 255 { + return []string{encoded}, nil + } + + // split encoded string + var record []string + + for len(encoded) > 255 { + record = append(record, encoded[:255]) + encoded = encoded[255:] + } + + record = append(record, encoded) + + return record, nil +} + +func decode(record []string) (*mdnsTxt, error) { + encoded := strings.Join(record, "") + + hr, err := hex.DecodeString(encoded) + if err != nil { + return nil, err + } + + br := bytes.NewReader(hr) + zr, err := zlib.NewReader(br) + if err != nil { + return nil, err + } + + rbuf, err := ioutil.ReadAll(zr) + if err != nil { + return nil, err + } + + var txt *mdnsTxt + + if err := json.Unmarshal(rbuf, &txt); err != nil { + return nil, err + } + + return txt, nil +} func newRegistry(opts ...Option) Registry { options := Options{ Context: context.Background(), diff --git a/registry/mdns_test.go b/registry/mdns_test.go index a5630c67..c9979d5c 100644 --- a/registry/mdns_test.go +++ b/registry/mdns_test.go @@ -137,3 +137,63 @@ func TestMDNS(t *testing.T) { } } + +func TestEncoding(t *testing.T) { + testData := []*mdnsTxt{ + { + Version: "1.0.0", + Metadata: map[string]string{ + "foo": "bar", + }, + Endpoints: []*Endpoint{ + { + Name: "endpoint1", + Request: &Value{ + Name: "request", + Type: "request", + }, + Response: &Value{ + Name: "response", + Type: "response", + }, + Metadata: map[string]string{ + "foo1": "bar1", + }, + }, + }, + }, + } + + for _, d := range testData { + encoded, err := encode(d) + if err != nil { + t.Fatal(err) + } + + for _, txt := range encoded { + if len(txt) > 255 { + t.Fatalf("One of parts for txt is %d characters", len(txt)) + } + } + + decoded, err := decode(encoded) + if err != nil { + t.Fatal(err) + } + + if decoded.Version != d.Version { + t.Fatalf("Expected version %s got %s", d.Version, decoded.Version) + } + + if len(decoded.Endpoints) != len(d.Endpoints) { + t.Fatalf("Expected %d endpoints, got %d", len(d.Endpoints), len(decoded.Endpoints)) + } + + for k, v := range d.Metadata { + if val := decoded.Metadata[k]; val != v { + t.Fatalf("Expected %s=%s got %s=%s", k, v, k, val) + } + } + } + +} From e5268dd0a62a79787e60400e9d55467a6cabe4dc Mon Sep 17 00:00:00 2001 From: Asim Aslam Date: Fri, 10 Apr 2020 17:41:10 +0100 Subject: [PATCH 09/44] move reg util to own package (#1523) * move reg util to own package * fix test * fix broken static router --- api/router/static/static.go | 3 +- registry/cache/cache.go | 5 +-- {registry => util/registry}/util.go | 44 +++++++++++++----------- {registry => util/registry}/util_test.go | 16 +++++---- 4 files changed, 38 insertions(+), 30 deletions(-) rename {registry => util/registry}/util.go (64%) rename {registry => util/registry}/util_test.go (77%) diff --git a/api/router/static/static.go b/api/router/static/static.go index 09271c7c..f868e52d 100644 --- a/api/router/static/static.go +++ b/api/router/static/static.go @@ -16,6 +16,7 @@ import ( "github.com/micro/go-micro/v2/logger" "github.com/micro/go-micro/v2/metadata" "github.com/micro/go-micro/v2/registry" + util "github.com/micro/go-micro/v2/util/registry" ) type endpoint struct { @@ -164,7 +165,7 @@ func (r *staticRouter) Endpoint(req *http.Request) (*api.Service, error) { // hack for stream endpoint if ep.apiep.Stream { - svcs := registry.Copy(services) + svcs := util.Copy(services) for _, svc := range svcs { if len(svc.Endpoints) == 0 { e := ®istry.Endpoint{} diff --git a/registry/cache/cache.go b/registry/cache/cache.go index eb12d16d..7714f980 100644 --- a/registry/cache/cache.go +++ b/registry/cache/cache.go @@ -9,6 +9,7 @@ import ( "github.com/micro/go-micro/v2/logger" "github.com/micro/go-micro/v2/registry" + util "github.com/micro/go-micro/v2/util/registry" ) // Cache is the registry cache interface @@ -119,7 +120,7 @@ func (c *cache) get(service string) ([]*registry.Service, error) { // get cache ttl ttl := c.ttls[service] // make a copy - cp := registry.Copy(services) + cp := util.Copy(services) // got services && within ttl so return cache if c.isValid(cp, ttl) { @@ -152,7 +153,7 @@ func (c *cache) get(service string) ([]*registry.Service, error) { // cache results c.Lock() - c.set(service, registry.Copy(services)) + c.set(service, util.Copy(services)) c.Unlock() return services, nil diff --git a/registry/util.go b/util/registry/util.go similarity index 64% rename from registry/util.go rename to util/registry/util.go index 729d3ca8..140faedd 100644 --- a/registry/util.go +++ b/util/registry/util.go @@ -1,7 +1,11 @@ package registry -func addNodes(old, neu []*Node) []*Node { - nodes := make([]*Node, len(neu)) +import ( + "github.com/micro/go-micro/v2/registry" +) + +func addNodes(old, neu []*registry.Node) []*registry.Node { + nodes := make([]*registry.Node, len(neu)) // add all new nodes for i, n := range neu { node := *n @@ -31,8 +35,8 @@ func addNodes(old, neu []*Node) []*Node { return nodes } -func delNodes(old, del []*Node) []*Node { - var nodes []*Node +func delNodes(old, del []*registry.Node) []*registry.Node { + var nodes []*registry.Node for _, o := range old { var rem bool for _, n := range del { @@ -49,24 +53,24 @@ func delNodes(old, del []*Node) []*Node { } // CopyService make a copy of service -func CopyService(service *Service) *Service { +func CopyService(service *registry.Service) *registry.Service { // copy service - s := new(Service) + s := new(registry.Service) *s = *service // copy nodes - nodes := make([]*Node, len(service.Nodes)) + nodes := make([]*registry.Node, len(service.Nodes)) for j, node := range service.Nodes { - n := new(Node) + n := new(registry.Node) *n = *node nodes[j] = n } s.Nodes = nodes // copy endpoints - eps := make([]*Endpoint, len(service.Endpoints)) + eps := make([]*registry.Endpoint, len(service.Endpoints)) for j, ep := range service.Endpoints { - e := new(Endpoint) + e := new(registry.Endpoint) *e = *ep eps[j] = e } @@ -75,8 +79,8 @@ func CopyService(service *Service) *Service { } // Copy makes a copy of services -func Copy(current []*Service) []*Service { - services := make([]*Service, len(current)) +func Copy(current []*registry.Service) []*registry.Service { + services := make([]*registry.Service, len(current)) for i, service := range current { services[i] = CopyService(service) } @@ -84,14 +88,14 @@ func Copy(current []*Service) []*Service { } // Merge merges two lists of services and returns a new copy -func Merge(olist []*Service, nlist []*Service) []*Service { - var srv []*Service +func Merge(olist []*registry.Service, nlist []*registry.Service) []*registry.Service { + var srv []*registry.Service for _, n := range nlist { var seen bool for _, o := range olist { if o.Version == n.Version { - sp := new(Service) + sp := new(registry.Service) // make copy *sp = *o // set nodes @@ -102,25 +106,25 @@ func Merge(olist []*Service, nlist []*Service) []*Service { srv = append(srv, sp) break } else { - sp := new(Service) + sp := new(registry.Service) // make copy *sp = *o srv = append(srv, sp) } } if !seen { - srv = append(srv, Copy([]*Service{n})...) + srv = append(srv, Copy([]*registry.Service{n})...) } } return srv } // Remove removes services and returns a new copy -func Remove(old, del []*Service) []*Service { - var services []*Service +func Remove(old, del []*registry.Service) []*registry.Service { + var services []*registry.Service for _, o := range old { - srv := new(Service) + srv := new(registry.Service) *srv = *o var rem bool diff --git a/registry/util_test.go b/util/registry/util_test.go similarity index 77% rename from registry/util_test.go rename to util/registry/util_test.go index e1a6e9a0..740ab5b1 100644 --- a/registry/util_test.go +++ b/util/registry/util_test.go @@ -3,14 +3,16 @@ package registry import ( "os" "testing" + + "github.com/micro/go-micro/v2/registry" ) func TestRemove(t *testing.T) { - services := []*Service{ + services := []*registry.Service{ { Name: "foo", Version: "1.0.0", - Nodes: []*Node{ + Nodes: []*registry.Node{ { Id: "foo-123", Address: "localhost:9999", @@ -20,7 +22,7 @@ func TestRemove(t *testing.T) { { Name: "foo", Version: "1.0.0", - Nodes: []*Node{ + Nodes: []*registry.Node{ { Id: "foo-123", Address: "localhost:6666", @@ -29,7 +31,7 @@ func TestRemove(t *testing.T) { }, } - servs := Remove([]*Service{services[0]}, []*Service{services[1]}) + servs := Remove([]*registry.Service{services[0]}, []*registry.Service{services[1]}) if i := len(servs); i > 0 { t.Errorf("Expected 0 nodes, got %d: %+v", i, servs) } @@ -39,11 +41,11 @@ func TestRemove(t *testing.T) { } func TestRemoveNodes(t *testing.T) { - services := []*Service{ + services := []*registry.Service{ { Name: "foo", Version: "1.0.0", - Nodes: []*Node{ + Nodes: []*registry.Node{ { Id: "foo-123", Address: "localhost:9999", @@ -57,7 +59,7 @@ func TestRemoveNodes(t *testing.T) { { Name: "foo", Version: "1.0.0", - Nodes: []*Node{ + Nodes: []*registry.Node{ { Id: "foo-123", Address: "localhost:6666", From 57853b2849aa484d72e91a9522563f3fa1a2ef7e Mon Sep 17 00:00:00 2001 From: Asim Aslam Date: Fri, 10 Apr 2020 17:43:02 +0100 Subject: [PATCH 10/44] remove etcd store --- store/etcd/config.go | 178 -------------------------- store/etcd/etcd.go | 272 ---------------------------------------- store/etcd/etcd_test.go | 225 --------------------------------- 3 files changed, 675 deletions(-) delete mode 100644 store/etcd/config.go delete mode 100644 store/etcd/etcd.go delete mode 100644 store/etcd/etcd_test.go diff --git a/store/etcd/config.go b/store/etcd/config.go deleted file mode 100644 index eb79cea1..00000000 --- a/store/etcd/config.go +++ /dev/null @@ -1,178 +0,0 @@ -package etcd - -import ( - "context" - cryptotls "crypto/tls" - "time" - - "github.com/coreos/etcd/clientv3" - "github.com/micro/go-micro/v2/store" - "google.golang.org/grpc" -) - -// Implement all the options from https://pkg.go.dev/github.com/coreos/etcd/clientv3?tab=doc#Config -// Need to use non basic types in context.WithValue -type autoSyncInterval string -type dialTimeout string -type dialKeepAliveTime string -type dialKeepAliveTimeout string -type maxCallSendMsgSize string -type maxCallRecvMsgSize string -type tls string -type username string -type password string -type rejectOldCluster string -type dialOptions string -type clientContext string -type permitWithoutStream string - -// AutoSyncInterval is the interval to update endpoints with its latest members. -// 0 disables auto-sync. By default auto-sync is disabled. -func AutoSyncInterval(d time.Duration) store.Option { - return func(o *store.Options) { - o.Context = context.WithValue(o.Context, autoSyncInterval(""), d) - } -} - -// DialTimeout is the timeout for failing to establish a connection. -func DialTimeout(d time.Duration) store.Option { - return func(o *store.Options) { - o.Context = context.WithValue(o.Context, dialTimeout(""), d) - } -} - -// DialKeepAliveTime is the time after which client pings the server to see if -// transport is alive. -func DialKeepAliveTime(d time.Duration) store.Option { - return func(o *store.Options) { - o.Context = context.WithValue(o.Context, dialKeepAliveTime(""), d) - } -} - -// DialKeepAliveTimeout is the time that the client waits for a response for the -// keep-alive probe. If the response is not received in this time, the connection is closed. -func DialKeepAliveTimeout(d time.Duration) store.Option { - return func(o *store.Options) { - o.Context = context.WithValue(o.Context, dialKeepAliveTimeout(""), d) - } -} - -// MaxCallSendMsgSize is the client-side request send limit in bytes. -// If 0, it defaults to 2.0 MiB (2 * 1024 * 1024). -// Make sure that "MaxCallSendMsgSize" < server-side default send/recv limit. -// ("--max-request-bytes" flag to etcd or "embed.Config.MaxRequestBytes"). -func MaxCallSendMsgSize(size int) store.Option { - return func(o *store.Options) { - o.Context = context.WithValue(o.Context, maxCallSendMsgSize(""), size) - } -} - -// MaxCallRecvMsgSize is the client-side response receive limit. -// If 0, it defaults to "math.MaxInt32", because range response can -// easily exceed request send limits. -// Make sure that "MaxCallRecvMsgSize" >= server-side default send/recv limit. -// ("--max-request-bytes" flag to etcd or "embed.Config.MaxRequestBytes"). -func MaxCallRecvMsgSize(size int) store.Option { - return func(o *store.Options) { - o.Context = context.WithValue(o.Context, maxCallRecvMsgSize(""), size) - } -} - -// TLS holds the client secure credentials, if any. -func TLS(conf *cryptotls.Config) store.Option { - return func(o *store.Options) { - t := conf.Clone() - o.Context = context.WithValue(o.Context, tls(""), t) - } -} - -// Username is a user name for authentication. -func Username(u string) store.Option { - return func(o *store.Options) { - o.Context = context.WithValue(o.Context, username(""), u) - } -} - -// Password is a password for authentication. -func Password(p string) store.Option { - return func(o *store.Options) { - o.Context = context.WithValue(o.Context, password(""), p) - } -} - -// RejectOldCluster when set will refuse to create a client against an outdated cluster. -func RejectOldCluster(b bool) store.Option { - return func(o *store.Options) { - o.Context = context.WithValue(o.Context, rejectOldCluster(""), b) - } -} - -// DialOptions is a list of dial options for the grpc client (e.g., for interceptors). -// For example, pass "grpc.WithBlock()" to block until the underlying connection is up. -// Without this, Dial returns immediately and connecting the server happens in background. -func DialOptions(opts []grpc.DialOption) store.Option { - return func(o *store.Options) { - if len(opts) > 0 { - ops := make([]grpc.DialOption, len(opts)) - copy(ops, opts) - o.Context = context.WithValue(o.Context, dialOptions(""), ops) - } - } -} - -// ClientContext is the default etcd3 client context; it can be used to cancel grpc -// dial out andother operations that do not have an explicit context. -func ClientContext(ctx context.Context) store.Option { - return func(o *store.Options) { - o.Context = context.WithValue(o.Context, clientContext(""), ctx) - } -} - -// PermitWithoutStream when set will allow client to send keepalive pings to server without any active streams(RPCs). -func PermitWithoutStream(b bool) store.Option { - return func(o *store.Options) { - o.Context = context.WithValue(o.Context, permitWithoutStream(""), b) - } -} - -func (e *etcdStore) applyConfig(cfg *clientv3.Config) { - if v := e.options.Context.Value(autoSyncInterval("")); v != nil { - cfg.AutoSyncInterval = v.(time.Duration) - } - if v := e.options.Context.Value(dialTimeout("")); v != nil { - cfg.DialTimeout = v.(time.Duration) - } - if v := e.options.Context.Value(dialKeepAliveTime("")); v != nil { - cfg.DialKeepAliveTime = v.(time.Duration) - } - if v := e.options.Context.Value(dialKeepAliveTimeout("")); v != nil { - cfg.DialKeepAliveTimeout = v.(time.Duration) - } - if v := e.options.Context.Value(maxCallSendMsgSize("")); v != nil { - cfg.MaxCallSendMsgSize = v.(int) - } - if v := e.options.Context.Value(maxCallRecvMsgSize("")); v != nil { - cfg.MaxCallRecvMsgSize = v.(int) - } - if v := e.options.Context.Value(tls("")); v != nil { - cfg.TLS = v.(*cryptotls.Config) - } - if v := e.options.Context.Value(username("")); v != nil { - cfg.Username = v.(string) - } - if v := e.options.Context.Value(password("")); v != nil { - cfg.Username = v.(string) - } - if v := e.options.Context.Value(rejectOldCluster("")); v != nil { - cfg.RejectOldCluster = v.(bool) - } - if v := e.options.Context.Value(dialOptions("")); v != nil { - cfg.DialOptions = v.([]grpc.DialOption) - } - if v := e.options.Context.Value(clientContext("")); v != nil { - cfg.Context = v.(context.Context) - } - if v := e.options.Context.Value(permitWithoutStream("")); v != nil { - cfg.PermitWithoutStream = v.(bool) - } -} diff --git a/store/etcd/etcd.go b/store/etcd/etcd.go deleted file mode 100644 index d4d9cf97..00000000 --- a/store/etcd/etcd.go +++ /dev/null @@ -1,272 +0,0 @@ -// Package etcd implements a go-micro/v2/store with etcd -package etcd - -import ( - "bytes" - "context" - "encoding/gob" - "math" - "strings" - "time" - - "github.com/coreos/etcd/clientv3" - "github.com/coreos/etcd/clientv3/namespace" - "github.com/micro/go-micro/v2/store" - "github.com/pkg/errors" -) - -type etcdStore struct { - options store.Options - - client *clientv3.Client - config clientv3.Config -} - -// NewStore returns a new etcd store -func NewStore(opts ...store.Option) store.Store { - e := &etcdStore{} - for _, o := range opts { - o(&e.options) - } - e.init() - return e -} - -func (e *etcdStore) Close() error { - return e.client.Close() -} - -func (e *etcdStore) Init(opts ...store.Option) error { - for _, o := range opts { - o(&e.options) - } - return e.init() -} - -func (e *etcdStore) init() error { - // ensure context is non-nil - e.options.Context = context.Background() - // set up config - e.config = clientv3.Config{} - e.applyConfig(&e.config) - if len(e.options.Nodes) == 0 { - e.config.Endpoints = []string{"http://127.0.0.1:2379"} - } else { - e.config.Endpoints = make([]string, len(e.options.Nodes)) - copy(e.config.Endpoints, e.options.Nodes) - } - if e.client != nil { - e.client.Close() - } - client, err := clientv3.New(e.config) - if err != nil { - return err - } - e.client = client - ns := "" - if len(e.options.Table) > 0 { - ns = e.options.Table - } - if len(e.options.Database) > 0 { - ns = e.options.Database + "/" + ns - } - if len(ns) > 0 { - e.client.KV = namespace.NewKV(e.client.KV, ns) - e.client.Watcher = namespace.NewWatcher(e.client.Watcher, ns) - e.client.Lease = namespace.NewLease(e.client.Lease, ns) - } - - return nil -} - -func (e *etcdStore) Options() store.Options { - return e.options -} - -func (e *etcdStore) String() string { - return "etcd" -} - -func (e *etcdStore) Read(key string, opts ...store.ReadOption) ([]*store.Record, error) { - readOpts := store.ReadOptions{} - for _, o := range opts { - o(&readOpts) - } - if readOpts.Suffix { - return e.readSuffix(key, readOpts) - } - - var etcdOpts []clientv3.OpOption - if readOpts.Prefix { - etcdOpts = append(etcdOpts, clientv3.WithPrefix(), clientv3.WithSort(clientv3.SortByKey, clientv3.SortAscend)) - } - resp, err := e.client.KV.Get(context.Background(), key, etcdOpts...) - if err != nil { - return nil, err - } - if resp.Count == 0 && !(readOpts.Prefix || readOpts.Suffix) { - return nil, store.ErrNotFound - } - var records []*store.Record - for _, kv := range resp.Kvs { - ir := internalRecord{} - if err := gob.NewDecoder(bytes.NewReader(kv.Value)).Decode(&ir); err != nil { - return records, errors.Wrapf(err, "couldn't decode %s into internalRecord", err.Error()) - } - r := store.Record{ - Key: ir.Key, - Value: ir.Value, - } - if !ir.ExpiresAt.IsZero() { - r.Expiry = time.Until(ir.ExpiresAt) - } - records = append(records, &r) - } - if readOpts.Limit > 0 || readOpts.Offset > 0 { - return records[readOpts.Offset:min(readOpts.Limit, uint(len(records)))], nil - } - return records, nil -} - -func (e *etcdStore) readSuffix(key string, readOpts store.ReadOptions) ([]*store.Record, error) { - opts := []store.ListOption{store.ListSuffix(key)} - if readOpts.Prefix { - opts = append(opts, store.ListPrefix(key)) - } - keys, err := e.List(opts...) - if err != nil { - return nil, errors.Wrapf(err, "Couldn't list with suffix %s", key) - } - var records []*store.Record - for _, k := range keys { - resp, err := e.client.KV.Get(context.Background(), k) - if err != nil { - return nil, errors.Wrapf(err, "Couldn't get key %s", k) - } - ir := internalRecord{} - if err := gob.NewDecoder(bytes.NewReader(resp.Kvs[0].Value)).Decode(&ir); err != nil { - return records, errors.Wrapf(err, "couldn't decode %s into internalRecord", err.Error()) - } - r := store.Record{ - Key: ir.Key, - Value: ir.Value, - } - if !ir.ExpiresAt.IsZero() { - r.Expiry = time.Until(ir.ExpiresAt) - } - records = append(records, &r) - - } - if readOpts.Limit > 0 || readOpts.Offset > 0 { - return records[readOpts.Offset:min(readOpts.Limit, uint(len(records)))], nil - } - return records, nil -} - -func (e *etcdStore) Write(r *store.Record, opts ...store.WriteOption) error { - options := store.WriteOptions{} - for _, o := range opts { - o(&options) - } - - if len(opts) > 0 { - // Copy the record before applying options, or the incoming record will be mutated - newRecord := store.Record{} - newRecord.Key = r.Key - newRecord.Value = make([]byte, len(r.Value)) - copy(newRecord.Value, r.Value) - newRecord.Expiry = r.Expiry - - if !options.Expiry.IsZero() { - newRecord.Expiry = time.Until(options.Expiry) - } - if options.TTL != 0 { - newRecord.Expiry = options.TTL - } - return e.write(&newRecord) - } - return e.write(r) -} - -func (e *etcdStore) write(r *store.Record) error { - var putOpts []clientv3.OpOption - ir := &internalRecord{} - ir.Key = r.Key - ir.Value = make([]byte, len(r.Value)) - copy(ir.Value, r.Value) - if r.Expiry != 0 { - ir.ExpiresAt = time.Now().Add(r.Expiry) - var leasexpiry int64 - if r.Expiry.Seconds() < 5.0 { - // minimum etcd lease is 5 seconds - leasexpiry = 5 - } else { - leasexpiry = int64(math.Ceil(r.Expiry.Seconds())) - } - lr, err := e.client.Lease.Grant(context.Background(), leasexpiry) - if err != nil { - return errors.Wrapf(err, "couldn't grant an etcd lease for %s", r.Key) - } - putOpts = append(putOpts, clientv3.WithLease(lr.ID)) - } - b := &bytes.Buffer{} - if err := gob.NewEncoder(b).Encode(ir); err != nil { - return errors.Wrapf(err, "couldn't encode %s", r.Key) - } - _, err := e.client.KV.Put(context.Background(), ir.Key, string(b.Bytes()), putOpts...) - return errors.Wrapf(err, "couldn't put key %s in to etcd", err) -} - -func (e *etcdStore) Delete(key string, opts ...store.DeleteOption) error { - options := store.DeleteOptions{} - for _, o := range opts { - o(&options) - } - _, err := e.client.KV.Delete(context.Background(), key) - return errors.Wrapf(err, "couldn't delete key %s", key) -} - -func (e *etcdStore) List(opts ...store.ListOption) ([]string, error) { - options := store.ListOptions{} - for _, o := range opts { - o(&options) - } - searchPrefix := "" - if len(options.Prefix) > 0 { - searchPrefix = options.Prefix - } - resp, err := e.client.KV.Get(context.Background(), searchPrefix, clientv3.WithPrefix(), clientv3.WithKeysOnly(), clientv3.WithSort(clientv3.SortByKey, clientv3.SortAscend)) - if err != nil { - return nil, errors.Wrap(err, "couldn't list, etcd get failed") - } - if len(options.Suffix) == 0 { - keys := make([]string, resp.Count) - for i, kv := range resp.Kvs { - keys[i] = string(kv.Key) - } - return keys, nil - } - keys := []string{} - for _, kv := range resp.Kvs { - if strings.HasSuffix(string(kv.Key), options.Suffix) { - keys = append(keys, string(kv.Key)) - } - } - if options.Limit > 0 || options.Offset > 0 { - return keys[options.Offset:min(options.Limit, uint(len(keys)))], nil - } - return keys, nil -} - -type internalRecord struct { - Key string - Value []byte - ExpiresAt time.Time -} - -func min(i, j uint) uint { - if i < j { - return i - } - return j -} diff --git a/store/etcd/etcd_test.go b/store/etcd/etcd_test.go deleted file mode 100644 index fade2fce..00000000 --- a/store/etcd/etcd_test.go +++ /dev/null @@ -1,225 +0,0 @@ -package etcd - -import ( - "fmt" - "testing" - "time" - - "github.com/kr/pretty" - "github.com/micro/go-micro/v2/store" -) - -func TestEtcd(t *testing.T) { - e := NewStore() - if err := e.Init(); err != nil { - t.Fatal(err) - } - //basictest(e, t) -} - -func basictest(s store.Store, t *testing.T) { - t.Logf("Testing store %s, with options %# v\n", s.String(), pretty.Formatter(s.Options())) - // Read and Write an expiring Record - if err := s.Write(&store.Record{ - Key: "Hello", - Value: []byte("World"), - Expiry: time.Second * 5, - }); err != nil { - t.Fatal(err) - } - if r, err := s.Read("Hello"); err != nil { - t.Fatal(err) - } else { - if len(r) != 1 { - t.Fatal("Read returned multiple records") - } - if r[0].Key != "Hello" { - t.Fatalf("Expected %s, got %s", "Hello", r[0].Key) - } - if string(r[0].Value) != "World" { - t.Fatalf("Expected %s, got %s", "World", r[0].Value) - } - } - time.Sleep(time.Second * 6) - if records, err := s.Read("Hello"); err != store.ErrNotFound { - t.Fatalf("Expected %# v, got %# v\nResults were %# v", store.ErrNotFound, err, pretty.Formatter(records)) - } - - // Write 3 records with various expiry and get with prefix - records := []*store.Record{ - &store.Record{ - Key: "foo", - Value: []byte("foofoo"), - }, - &store.Record{ - Key: "foobar", - Value: []byte("foobarfoobar"), - Expiry: time.Second * 5, - }, - &store.Record{ - Key: "foobarbaz", - Value: []byte("foobarbazfoobarbaz"), - Expiry: 2 * time.Second * 5, - }, - } - for _, r := range records { - if err := s.Write(r); err != nil { - t.Fatalf("Couldn't write k: %s, v: %# v (%s)", r.Key, pretty.Formatter(r.Value), err) - } - } - if results, err := s.Read("foo", store.ReadPrefix()); err != nil { - t.Fatalf("Couldn't read all \"foo\" keys, got %# v (%s)", pretty.Formatter(results), err) - } else { - if len(results) != 3 { - t.Fatalf("Expected 3 items, got %d", len(results)) - } - } - time.Sleep(time.Second * 6) - if results, err := s.Read("foo", store.ReadPrefix()); err != nil { - t.Fatalf("Couldn't read all \"foo\" keys, got %# v (%s)", pretty.Formatter(results), err) - } else { - if len(results) != 2 { - t.Fatalf("Expected 2 items, got %d", len(results)) - } - } - time.Sleep(time.Second * 5) - if results, err := s.Read("foo", store.ReadPrefix()); err != nil { - t.Fatalf("Couldn't read all \"foo\" keys, got %# v (%s)", pretty.Formatter(results), err) - } else { - if len(results) != 1 { - t.Fatalf("Expected 1 item, got %d", len(results)) - } - } - if err := s.Delete("foo", func(d *store.DeleteOptions) {}); err != nil { - t.Fatalf("Delete failed (%v)", err) - } - if results, err := s.Read("foo", store.ReadPrefix()); err != nil { - t.Fatalf("Couldn't read all \"foo\" keys, got %# v (%s)", pretty.Formatter(results), err) - } else { - if len(results) != 0 { - t.Fatalf("Expected 0 items, got %d (%# v)", len(results), pretty.Formatter(results)) - } - } - - // Write 3 records with various expiry and get with Suffix - records = []*store.Record{ - &store.Record{ - Key: "foo", - Value: []byte("foofoo"), - }, - &store.Record{ - Key: "barfoo", - Value: []byte("barfoobarfoo"), - Expiry: time.Second * 5, - }, - &store.Record{ - Key: "bazbarfoo", - Value: []byte("bazbarfoobazbarfoo"), - Expiry: 2 * time.Second * 5, - }, - } - for _, r := range records { - if err := s.Write(r); err != nil { - t.Fatalf("Couldn't write k: %s, v: %# v (%s)", r.Key, pretty.Formatter(r.Value), err) - } - } - if results, err := s.Read("foo", store.ReadSuffix()); err != nil { - t.Fatalf("Couldn't read all \"foo\" keys, got %# v (%s)", pretty.Formatter(results), err) - } else { - if len(results) != 3 { - t.Fatalf("Expected 3 items, got %d", len(results)) - } - } - time.Sleep(time.Second * 6) - if results, err := s.Read("foo", store.ReadSuffix()); err != nil { - t.Fatalf("Couldn't read all \"foo\" keys, got %# v (%s)", pretty.Formatter(results), err) - } else { - if len(results) != 2 { - t.Fatalf("Expected 2 items, got %d", len(results)) - } - t.Logf("Prefix test: %v\n", pretty.Formatter(results)) - } - time.Sleep(time.Second * 5) - if results, err := s.Read("foo", store.ReadSuffix()); err != nil { - t.Fatalf("Couldn't read all \"foo\" keys, got %# v (%s)", pretty.Formatter(results), err) - } else { - if len(results) != 1 { - t.Fatalf("Expected 1 item, got %d", len(results)) - } - t.Logf("Prefix test: %# v\n", pretty.Formatter(results)) - } - if err := s.Delete("foo"); err != nil { - t.Fatalf("Delete failed (%v)", err) - } - if results, err := s.Read("foo", store.ReadSuffix()); err != nil { - t.Fatalf("Couldn't read all \"foo\" keys, got %# v (%s)", pretty.Formatter(results), err) - } else { - if len(results) != 0 { - t.Fatalf("Expected 0 items, got %d (%# v)", len(results), pretty.Formatter(results)) - } - } - - // Test Prefix, Suffix and WriteOptions - if err := s.Write(&store.Record{ - Key: "foofoobarbar", - Value: []byte("something"), - }, store.WriteTTL(time.Millisecond*100)); err != nil { - t.Fatal(err) - } - if err := s.Write(&store.Record{ - Key: "foofoo", - Value: []byte("something"), - }, store.WriteExpiry(time.Now().Add(time.Millisecond*100))); err != nil { - t.Fatal(err) - } - if err := s.Write(&store.Record{ - Key: "barbar", - Value: []byte("something"), - // TTL has higher precedence than expiry - }, store.WriteExpiry(time.Now().Add(time.Hour)), store.WriteTTL(time.Millisecond*100)); err != nil { - t.Fatal(err) - } - if results, err := s.Read("foo", store.ReadPrefix(), store.ReadSuffix()); err != nil { - t.Fatal(err) - } else { - if len(results) != 1 { - t.Fatalf("Expected 1 results, got %d: %# v", len(results), pretty.Formatter(results)) - } - } - time.Sleep(time.Second * 6) - if results, err := s.List(); err != nil { - t.Fatalf("List failed: %s", err) - } else { - if len(results) != 0 { - t.Fatal("Expiry options were not effective") - } - } - - s.Init() - for i := 0; i < 10; i++ { - s.Write(&store.Record{ - Key: fmt.Sprintf("a%d", i), - Value: []byte{}, - }) - } - if results, err := s.Read("a", store.ReadLimit(5), store.ReadPrefix()); err != nil { - t.Fatal(err) - } else { - if len(results) != 5 { - t.Fatal("Expected 5 results, got ", len(results)) - } - if results[0].Key != "a0" { - t.Fatalf("Expected a0, got %s", results[0].Key) - } - if results[4].Key != "a4" { - t.Fatalf("Expected a4, got %s", results[4].Key) - } - } - if results, err := s.Read("a", store.ReadLimit(30), store.ReadOffset(5), store.ReadPrefix()); err != nil { - t.Fatal(err) - } else { - if len(results) != 5 { - t.Fatal("Expected 5 results, got ", len(results)) - } - } -} From b9a5e9d61013b7bde33235903b1b379063e17de5 Mon Sep 17 00:00:00 2001 From: Asim Aslam Date: Fri, 10 Apr 2020 17:47:13 +0100 Subject: [PATCH 11/44] fixup sync map --- sync/map.go | 10 ---------- 1 file changed, 10 deletions(-) diff --git a/sync/map.go b/sync/map.go index edfeb6a9..2196a4e0 100644 --- a/sync/map.go +++ b/sync/map.go @@ -8,8 +8,6 @@ import ( "sort" "github.com/micro/go-micro/v2/store" - ckv "github.com/micro/go-micro/v2/store/etcd" - lock "github.com/micro/go-micro/v2/sync/lock/etcd" ) type syncMap struct { @@ -152,14 +150,6 @@ func NewMap(opts ...Option) Map { o(&options) } - if options.Lock == nil { - options.Lock = lock.NewLock() - } - - if options.Store == nil { - options.Store = ckv.NewStore() - } - return &syncMap{ opts: options, } From d4b2c948ddecf921660f2a98aceae0e38784e042 Mon Sep 17 00:00:00 2001 From: Asim Aslam Date: Fri, 10 Apr 2020 19:50:57 +0100 Subject: [PATCH 12/44] Remove cloudflare store --- api/server/acme/certmagic/certmagic_test.go | 236 ----------- store/cloudflare/cloudflare.go | 411 -------------------- store/cloudflare/cloudflare_test.go | 95 ----- store/cloudflare/options.go | 64 --- 4 files changed, 806 deletions(-) delete mode 100644 api/server/acme/certmagic/certmagic_test.go delete mode 100644 store/cloudflare/cloudflare.go delete mode 100644 store/cloudflare/cloudflare_test.go delete mode 100644 store/cloudflare/options.go diff --git a/api/server/acme/certmagic/certmagic_test.go b/api/server/acme/certmagic/certmagic_test.go deleted file mode 100644 index 12893dfe..00000000 --- a/api/server/acme/certmagic/certmagic_test.go +++ /dev/null @@ -1,236 +0,0 @@ -package certmagic - -import ( - "net" - "net/http" - "os" - "reflect" - "sort" - "testing" - "time" - - "github.com/go-acme/lego/v3/providers/dns/cloudflare" - "github.com/mholt/certmagic" - "github.com/micro/go-micro/v2/api/server/acme" - cfstore "github.com/micro/go-micro/v2/store/cloudflare" - "github.com/micro/go-micro/v2/sync/lock/memory" -) - -func TestCertMagic(t *testing.T) { - if len(os.Getenv("IN_TRAVIS_CI")) != 0 { - t.Skip() - } - l, err := NewProvider().Listen() - if err != nil { - if _, ok := err.(*net.OpError); ok { - t.Skip("Run under non privileged user") - } - t.Fatal(err.Error()) - } - l.Close() - - c := cloudflare.NewDefaultConfig() - c.AuthEmail = "" - c.AuthKey = "" - c.AuthToken = "test" - c.ZoneToken = "test" - - p, err := cloudflare.NewDNSProviderConfig(c) - if err != nil { - t.Fatal(err.Error()) - } - - l, err = NewProvider(acme.AcceptToS(true), - acme.CA(acme.LetsEncryptStagingCA), - acme.ChallengeProvider(p), - ).Listen() - - if err != nil { - t.Fatal(err.Error()) - } - l.Close() -} - -func TestStorageImplementation(t *testing.T) { - if len(os.Getenv("IN_TRAVIS_CI")) != 0 { - t.Skip() - } - - apiToken, accountID := os.Getenv("CF_API_TOKEN"), os.Getenv("CF_ACCOUNT_ID") - kvID := os.Getenv("KV_NAMESPACE_ID") - if len(apiToken) == 0 || len(accountID) == 0 || len(kvID) == 0 { - t.Skip("No Cloudflare API keys available, skipping test") - } - - var s certmagic.Storage - st := cfstore.NewStore( - cfstore.Token(apiToken), - cfstore.Account(accountID), - cfstore.Namespace(kvID), - ) - s = &storage{ - lock: memory.NewLock(), - store: st, - } - - // Test Lock - if err := s.Lock("test"); err != nil { - t.Fatal(err) - } - - // Test Unlock - if err := s.Unlock("test"); err != nil { - t.Fatal(err) - } - - // Test data - testdata := []struct { - key string - value []byte - }{ - {key: "/foo/a", value: []byte("lorem")}, - {key: "/foo/b", value: []byte("ipsum")}, - {key: "/foo/c", value: []byte("dolor")}, - {key: "/foo/d", value: []byte("sit")}, - {key: "/bar/a", value: []byte("amet")}, - {key: "/bar/b", value: []byte("consectetur")}, - {key: "/bar/c", value: []byte("adipiscing")}, - {key: "/bar/d", value: []byte("elit")}, - {key: "/foo/bar/a", value: []byte("sed")}, - {key: "/foo/bar/b", value: []byte("do")}, - {key: "/foo/bar/c", value: []byte("eiusmod")}, - {key: "/foo/bar/d", value: []byte("tempor")}, - {key: "/foo/bar/baz/a", value: []byte("incididunt")}, - {key: "/foo/bar/baz/b", value: []byte("ut")}, - {key: "/foo/bar/baz/c", value: []byte("labore")}, - {key: "/foo/bar/baz/d", value: []byte("et")}, - // a duplicate just in case there's any edge cases - {key: "/foo/a", value: []byte("lorem")}, - } - - // Test Store - for _, d := range testdata { - if err := s.Store(d.key, d.value); err != nil { - t.Fatal(err.Error()) - } - } - - // Test Load - for _, d := range testdata { - if value, err := s.Load(d.key); err != nil { - t.Fatal(err.Error()) - } else { - if !reflect.DeepEqual(value, d.value) { - t.Fatalf("Load %s: expected %v, got %v", d.key, d.value, value) - } - } - } - - // Test Exists - for _, d := range testdata { - if !s.Exists(d.key) { - t.Fatalf("%s should exist, but doesn't\n", d.key) - } - } - - // Test List - if list, err := s.List("/", true); err != nil { - t.Fatal(err.Error()) - } else { - var expected []string - for i, d := range testdata { - if i != len(testdata)-1 { - // Don't store the intentionally duplicated key - expected = append(expected, d.key) - } - } - sort.Strings(expected) - sort.Strings(list) - if !reflect.DeepEqual(expected, list) { - t.Fatalf("List: Expected %v, got %v\n", expected, list) - } - } - if list, err := s.List("/foo", false); err != nil { - t.Fatal(err.Error()) - } else { - sort.Strings(list) - expected := []string{"/foo/a", "/foo/b", "/foo/bar", "/foo/c", "/foo/d"} - if !reflect.DeepEqual(expected, list) { - t.Fatalf("List: expected %s, got %s\n", expected, list) - } - } - - // Test Stat - for _, d := range testdata { - info, err := s.Stat(d.key) - if err != nil { - t.Fatal(err.Error()) - } else { - if info.Key != d.key { - t.Fatalf("Stat().Key: expected %s, got %s\n", d.key, info.Key) - } - if info.Size != int64(len(d.value)) { - t.Fatalf("Stat().Size: expected %d, got %d\n", len(d.value), info.Size) - } - if time.Since(info.Modified) > time.Minute { - t.Fatalf("Stat().Modified: expected time since last modified to be < 1 minute, got %v\n", time.Since(info.Modified)) - } - } - - } - - // Test Delete - for _, d := range testdata { - if err := s.Delete(d.key); err != nil { - t.Fatal(err.Error()) - } - } - - // New interface doesn't return an error, so call it in case any log.Fatal - // happens - NewProvider(acme.Cache(s)) -} - -// Full test with a real zone, with against LE staging -func TestE2e(t *testing.T) { - if len(os.Getenv("IN_TRAVIS_CI")) != 0 { - t.Skip() - } - - apiToken, accountID := os.Getenv("CF_API_TOKEN"), os.Getenv("CF_ACCOUNT_ID") - kvID := os.Getenv("KV_NAMESPACE_ID") - if len(apiToken) == 0 || len(accountID) == 0 || len(kvID) == 0 { - t.Skip("No Cloudflare API keys available, skipping test") - } - - testLock := memory.NewLock() - testStore := cfstore.NewStore( - cfstore.Token(apiToken), - cfstore.Account(accountID), - cfstore.Namespace(kvID), - ) - testStorage := NewStorage(testLock, testStore) - - conf := cloudflare.NewDefaultConfig() - conf.AuthToken = apiToken - conf.ZoneToken = apiToken - testChallengeProvider, err := cloudflare.NewDNSProviderConfig(conf) - if err != nil { - t.Fatal(err.Error()) - } - - testProvider := NewProvider( - acme.AcceptToS(true), - acme.Cache(testStorage), - acme.CA(acme.LetsEncryptStagingCA), - acme.ChallengeProvider(testChallengeProvider), - acme.OnDemand(false), - ) - - listener, err := testProvider.Listen("*.micro.mu", "micro.mu") - if err != nil { - t.Fatal(err.Error()) - } - go http.Serve(listener, http.NotFoundHandler()) - time.Sleep(10 * time.Minute) -} diff --git a/store/cloudflare/cloudflare.go b/store/cloudflare/cloudflare.go deleted file mode 100644 index cf3002f6..00000000 --- a/store/cloudflare/cloudflare.go +++ /dev/null @@ -1,411 +0,0 @@ -// Package cloudflare is a store implementation backed by cloudflare workers kv -// Note that the cloudflare workers KV API is eventually consistent. -package cloudflare - -import ( - "bytes" - "context" - "encoding/json" - "fmt" - "io" - "io/ioutil" - "log" - "math" - "net/http" - "net/url" - "os" - "strconv" - "strings" - "time" - - "github.com/micro/go-micro/v2/store" - "github.com/pkg/errors" - - "github.com/patrickmn/go-cache" -) - -const ( - apiBaseURL = "https://api.cloudflare.com/client/v4/" -) - -type workersKV struct { - options store.Options - // cf account id - account string - // cf api token - token string - // cf kv namespace - namespace string - // http client to use - httpClient *http.Client - // cache - cache *cache.Cache -} - -// apiResponse is a cloudflare v4 api response -type apiResponse struct { - Result []struct { - ID string `json:"id"` - Type string `json:"type"` - Name string `json:"name"` - Expiration int64 `json:"expiration"` - Content string `json:"content"` - Proxiable bool `json:"proxiable"` - Proxied bool `json:"proxied"` - TTL int64 `json:"ttl"` - Priority int64 `json:"priority"` - Locked bool `json:"locked"` - ZoneID string `json:"zone_id"` - ZoneName string `json:"zone_name"` - ModifiedOn time.Time `json:"modified_on"` - CreatedOn time.Time `json:"created_on"` - } `json:"result"` - Success bool `json:"success"` - Errors []apiMessage `json:"errors"` - // not sure Messages is ever populated? - Messages []apiMessage `json:"messages"` - ResultInfo struct { - Page int `json:"page"` - PerPage int `json:"per_page"` - Count int `json:"count"` - TotalCount int `json:"total_count"` - } `json:"result_info"` -} - -// apiMessage is a Cloudflare v4 API Error -type apiMessage struct { - Code int `json:"code"` - Message string `json:"message"` -} - -// getOptions returns account id, token and namespace -func getOptions() (string, string, string) { - accountID := strings.TrimSpace(os.Getenv("CF_ACCOUNT_ID")) - apiToken := strings.TrimSpace(os.Getenv("CF_API_TOKEN")) - namespace := strings.TrimSpace(os.Getenv("KV_NAMESPACE_ID")) - - return accountID, apiToken, namespace -} - -func validateOptions(account, token, namespace string) { - if len(account) == 0 { - log.Fatal("Store: CF_ACCOUNT_ID is blank") - } - - if len(token) == 0 { - log.Fatal("Store: CF_API_TOKEN is blank") - } - - if len(namespace) == 0 { - log.Fatal("Store: KV_NAMESPACE_ID is blank") - } -} - -func (w *workersKV) Close() error { - return nil -} - -func (w *workersKV) Init(opts ...store.Option) error { - for _, o := range opts { - o(&w.options) - } - if len(w.options.Database) > 0 { - w.namespace = w.options.Database - } - if w.options.Context == nil { - w.options.Context = context.TODO() - } - ttl := w.options.Context.Value("STORE_CACHE_TTL") - if ttl != nil { - ttlduration, ok := ttl.(time.Duration) - if !ok { - log.Fatal("STORE_CACHE_TTL from context must be type int64") - } - w.cache = cache.New(ttlduration, 3*ttlduration) - } - return nil -} - -func (w *workersKV) list(prefix string) ([]string, error) { - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() - - path := fmt.Sprintf("accounts/%s/storage/kv/namespaces/%s/keys", w.account, w.namespace) - - body := make(map[string]string) - - if len(prefix) > 0 { - body["prefix"] = prefix - } - - response, _, _, err := w.request(ctx, http.MethodGet, path, body, make(http.Header)) - if err != nil { - return nil, err - } - - a := &apiResponse{} - if err := json.Unmarshal(response, a); err != nil { - return nil, err - } - - if !a.Success { - messages := "" - for _, m := range a.Errors { - messages += strconv.Itoa(m.Code) + " " + m.Message + "\n" - } - return nil, errors.New(messages) - } - - keys := make([]string, 0, len(a.Result)) - - for _, r := range a.Result { - keys = append(keys, r.Name) - } - - return keys, nil -} - -// In the cloudflare workers KV implemention, List() doesn't guarantee -// anything as the workers API is eventually consistent. -func (w *workersKV) List(opts ...store.ListOption) ([]string, error) { - keys, err := w.list("") - if err != nil { - return nil, err - } - - return keys, nil -} - -func (w *workersKV) Read(key string, opts ...store.ReadOption) ([]*store.Record, error) { - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() - - var options store.ReadOptions - for _, o := range opts { - o(&options) - } - - keys := []string{key} - - if options.Prefix { - k, err := w.list(key) - if err != nil { - return nil, err - } - keys = k - } - - //nolint:prealloc - var records []*store.Record - - for _, k := range keys { - if w.cache != nil { - if resp, hit := w.cache.Get(k); hit { - if record, ok := resp.(*store.Record); ok { - records = append(records, record) - continue - } - } - } - - path := fmt.Sprintf("accounts/%s/storage/kv/namespaces/%s/values/%s", w.account, w.namespace, url.PathEscape(k)) - response, headers, status, err := w.request(ctx, http.MethodGet, path, nil, make(http.Header)) - if err != nil { - return records, err - } - if status < 200 || status >= 300 { - if status == 404 { - return nil, store.ErrNotFound - } - - return records, errors.New("Received unexpected Status " + strconv.Itoa(status) + string(response)) - } - record := &store.Record{ - Key: k, - Value: response, - } - if expiry := headers.Get("Expiration"); len(expiry) != 0 { - expiryUnix, err := strconv.ParseInt(expiry, 10, 64) - if err != nil { - return records, err - } - record.Expiry = time.Until(time.Unix(expiryUnix, 0)) - } - if w.cache != nil { - w.cache.Set(record.Key, record, cache.DefaultExpiration) - } - records = append(records, record) - } - - return records, nil -} - -func (w *workersKV) Write(r *store.Record, opts ...store.WriteOption) error { - // Set it in local cache, with the global TTL from options - if w.cache != nil { - w.cache.Set(r.Key, r, cache.DefaultExpiration) - } - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() - - path := fmt.Sprintf("accounts/%s/storage/kv/namespaces/%s/values/%s", w.account, w.namespace, url.PathEscape(r.Key)) - if r.Expiry != 0 { - // Minimum cloudflare TTL is 60 Seconds - exp := int(math.Max(60, math.Round(r.Expiry.Seconds()))) - path = path + "?expiration_ttl=" + strconv.Itoa(exp) - } - - headers := make(http.Header) - - resp, _, _, err := w.request(ctx, http.MethodPut, path, r.Value, headers) - if err != nil { - return err - } - - a := &apiResponse{} - if err := json.Unmarshal(resp, a); err != nil { - return err - } - - if !a.Success { - messages := "" - for _, m := range a.Errors { - messages += strconv.Itoa(m.Code) + " " + m.Message + "\n" - } - return errors.New(messages) - } - - return nil -} - -func (w *workersKV) Delete(key string, opts ...store.DeleteOption) error { - if w.cache != nil { - w.cache.Delete(key) - } - ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second) - defer cancel() - - path := fmt.Sprintf("accounts/%s/storage/kv/namespaces/%s/values/%s", w.account, w.namespace, url.PathEscape(key)) - resp, _, _, err := w.request(ctx, http.MethodDelete, path, nil, make(http.Header)) - if err != nil { - return err - } - - a := &apiResponse{} - if err := json.Unmarshal(resp, a); err != nil { - return err - } - - if !a.Success { - messages := "" - for _, m := range a.Errors { - messages += strconv.Itoa(m.Code) + " " + m.Message + "\n" - } - return errors.New(messages) - } - - return nil -} - -func (w *workersKV) request(ctx context.Context, method, path string, body interface{}, headers http.Header) ([]byte, http.Header, int, error) { - var jsonBody []byte - var err error - - if body != nil { - if paramBytes, ok := body.([]byte); ok { - jsonBody = paramBytes - } else { - jsonBody, err = json.Marshal(body) - if err != nil { - return nil, nil, 0, errors.Wrap(err, "error marshalling params to JSON") - } - } - } else { - jsonBody = nil - } - - var reqBody io.Reader - - if jsonBody != nil { - reqBody = bytes.NewReader(jsonBody) - } - - req, err := http.NewRequestWithContext(ctx, method, apiBaseURL+path, reqBody) - if err != nil { - return nil, nil, 0, errors.Wrap(err, "error creating new request") - } - - for key, value := range headers { - req.Header[key] = value - } - - // set token if it exists - if len(w.token) > 0 { - req.Header.Set("Authorization", "Bearer "+w.token) - } - - // set the user agent to micro - req.Header.Set("User-Agent", "micro/1.0 (https://micro.mu)") - - // Official cloudflare client does exponential backoff here - // TODO: retry and use util/backoff - resp, err := w.httpClient.Do(req) - if err != nil { - return nil, nil, 0, err - } - defer resp.Body.Close() - - respBody, err := ioutil.ReadAll(resp.Body) - if err != nil { - return respBody, resp.Header, resp.StatusCode, err - } - - return respBody, resp.Header, resp.StatusCode, nil -} - -func (w *workersKV) String() string { - return "cloudflare" -} - -func (w *workersKV) Options() store.Options { - return w.options -} - -// NewStore returns a cloudflare Store implementation. -// Account ID, Token and Namespace must either be passed as options or -// environment variables. If set as env vars we expect the following; -// CF_API_TOKEN to a cloudflare API token scoped to Workers KV. -// CF_ACCOUNT_ID to contain a string with your cloudflare account ID. -// KV_NAMESPACE_ID to contain the namespace UUID for your KV storage. -func NewStore(opts ...store.Option) store.Store { - var options store.Options - for _, o := range opts { - o(&options) - } - - // get options from environment - account, token, namespace := getOptions() - - if len(account) == 0 { - account = getAccount(options.Context) - } - - if len(token) == 0 { - token = getToken(options.Context) - } - - if len(namespace) == 0 { - namespace = options.Database - } - - // validate options are not blank or log.Fatal - validateOptions(account, token, namespace) - - return &workersKV{ - account: account, - namespace: namespace, - token: token, - options: options, - httpClient: &http.Client{}, - } -} diff --git a/store/cloudflare/cloudflare_test.go b/store/cloudflare/cloudflare_test.go deleted file mode 100644 index 9cc7a05b..00000000 --- a/store/cloudflare/cloudflare_test.go +++ /dev/null @@ -1,95 +0,0 @@ -package cloudflare - -import ( - "math/rand" - "os" - "strconv" - "testing" - "time" - - "github.com/micro/go-micro/v2/store" -) - -func TestCloudflare(t *testing.T) { - if len(os.Getenv("IN_TRAVIS_CI")) != 0 { - t.Skip() - } - - apiToken, accountID := os.Getenv("CF_API_TOKEN"), os.Getenv("CF_ACCOUNT_ID") - kvID := os.Getenv("KV_NAMESPACE_ID") - if len(apiToken) == 0 || len(accountID) == 0 || len(kvID) == 0 { - t.Skip("No Cloudflare API keys available, skipping test") - } - rand.Seed(time.Now().UnixNano()) - randomK := strconv.Itoa(rand.Int()) - randomV := strconv.Itoa(rand.Int()) - - wkv := NewStore( - Token(apiToken), - Account(accountID), - Namespace(kvID), - CacheTTL(60000000000), - ) - - records, err := wkv.List() - if err != nil { - t.Fatalf("List: %s\n", err.Error()) - } else { - if len(os.Getenv("IN_TRAVIS_CI")) == 0 { - t.Log("Listed " + strconv.Itoa(len(records)) + " records") - } - } - - err = wkv.Write(&store.Record{ - Key: randomK, - Value: []byte(randomV), - }) - if err != nil { - t.Errorf("Write: %s", err.Error()) - } - err = wkv.Write(&store.Record{ - Key: "expirationtest", - Value: []byte("This message will self destruct"), - Expiry: 75 * time.Second, - }) - if err != nil { - t.Errorf("Write: %s", err.Error()) - } - - // This might be needed for cloudflare eventual consistency - time.Sleep(1 * time.Minute) - - r, err := wkv.Read(randomK) - if err != nil { - t.Errorf("Read: %s\n", err.Error()) - } - if len(r) != 1 { - t.Errorf("Expected to read 1 key, got %d keys\n", len(r)) - } - if string(r[0].Value) != randomV { - t.Errorf("Read: expected %s, got %s\n", randomK, string(r[0].Value)) - } - - r, err = wkv.Read("expirationtest") - if err != nil { - t.Errorf("Read: expirationtest should still exist") - } - if r[0].Expiry == 0 { - t.Error("Expected r to have an expiry") - } else { - t.Log(r[0].Expiry) - } - - time.Sleep(20 * time.Second) - r, err = wkv.Read("expirationtest") - if err == nil && len(r) != 0 { - t.Error("Read: Managed to read expirationtest, but it should have expired") - t.Log(err, r[0].Key, string(r[0].Value), r[0].Expiry, len(r)) - } - - err = wkv.Delete(randomK) - if err != nil { - t.Errorf("Delete: %s\n", err.Error()) - } - -} diff --git a/store/cloudflare/options.go b/store/cloudflare/options.go deleted file mode 100644 index a0143dd8..00000000 --- a/store/cloudflare/options.go +++ /dev/null @@ -1,64 +0,0 @@ -package cloudflare - -import ( - "context" - "time" - - "github.com/micro/go-micro/v2/store" -) - -func getOption(ctx context.Context, key string) string { - if ctx == nil { - return "" - } - val, ok := ctx.Value(key).(string) - if !ok { - return "" - } - return val -} - -func getToken(ctx context.Context) string { - return getOption(ctx, "CF_API_TOKEN") -} - -func getAccount(ctx context.Context) string { - return getOption(ctx, "CF_ACCOUNT_ID") -} - -// Token sets the cloudflare api token -func Token(t string) store.Option { - return func(o *store.Options) { - if o.Context == nil { - o.Context = context.Background() - } - o.Context = context.WithValue(o.Context, "CF_API_TOKEN", t) - } -} - -// Account sets the cloudflare account id -func Account(id string) store.Option { - return func(o *store.Options) { - if o.Context == nil { - o.Context = context.Background() - } - o.Context = context.WithValue(o.Context, "CF_ACCOUNT_ID", id) - } -} - -// Namespace sets the KV namespace -func Namespace(ns string) store.Option { - return func(o *store.Options) { - o.Database = ns - } -} - -// CacheTTL sets the timeout in nanoseconds of the read/write cache -func CacheTTL(ttl time.Duration) store.Option { - return func(o *store.Options) { - if o.Context == nil { - o.Context = context.Background() - } - o.Context = context.WithValue(o.Context, "STORE_CACHE_TTL", ttl) - } -} From b5f546b137419898f8c1f1045e775b887572ab96 Mon Sep 17 00:00:00 2001 From: Asim Aslam Date: Fri, 10 Apr 2020 19:55:45 +0100 Subject: [PATCH 13/44] go mod tidy --- go.mod | 23 ++++++++++++++++++++--- go.sum | 50 +------------------------------------------------- 2 files changed, 21 insertions(+), 52 deletions(-) diff --git a/go.mod b/go.mod index 551a2982..4ce7a293 100644 --- a/go.mod +++ b/go.mod @@ -6,8 +6,14 @@ require ( github.com/BurntSushi/toml v0.3.1 github.com/beevik/ntp v0.2.0 github.com/bitly/go-simplejson v0.5.0 + github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 // indirect github.com/bwmarrin/discordgo v0.20.2 + github.com/coreos/bbolt v1.3.3 // indirect github.com/coreos/etcd v3.3.18+incompatible + github.com/coreos/go-semver v0.3.0 // indirect + github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f // indirect + github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f // indirect + github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect github.com/davecgh/go-spew v1.1.1 github.com/dgrijalva/jwt-go v3.2.0+incompatible github.com/ef-ds/deque v1.0.4-0.20190904040645-54cb57c252a1 @@ -17,25 +23,31 @@ require ( github.com/fsouza/go-dockerclient v1.6.0 github.com/ghodss/yaml v1.0.0 github.com/go-acme/lego/v3 v3.3.0 + github.com/go-telegram-bot-api/telegram-bot-api v4.6.4+incompatible // indirect github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee github.com/gobwas/pool v0.2.0 // indirect github.com/gobwas/ws v1.0.3 + github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect github.com/golang/protobuf v1.3.2 + github.com/google/go-cmp v0.4.0 // indirect github.com/google/uuid v1.1.1 github.com/gorilla/handlers v1.4.2 - github.com/gorilla/websocket v1.4.1 + github.com/gorilla/websocket v1.4.1 // indirect + github.com/grpc-ecosystem/go-grpc-middleware v1.1.0 // indirect + github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect github.com/grpc-ecosystem/grpc-gateway v1.9.5 github.com/hashicorp/hcl v1.0.0 github.com/hpcloud/tail v1.0.0 github.com/imdario/mergo v0.3.8 + github.com/jonboulle/clockwork v0.1.0 // indirect github.com/joncalhoun/qson v0.0.0-20170526102502-8a9cab3a62b1 + github.com/json-iterator/go v1.1.9 // indirect github.com/kr/pretty v0.1.0 github.com/lib/pq v1.3.0 github.com/lucas-clemente/quic-go v0.14.1 github.com/mholt/certmagic v0.9.3 github.com/micro/cli/v2 v2.1.2 github.com/micro/mdns v0.3.0 - github.com/micro/micro/v2 v2.4.0 github.com/miekg/dns v1.1.27 github.com/mitchellh/hashstructure v1.0.0 github.com/nats-io/nats-server/v2 v2.1.4 @@ -44,13 +56,18 @@ require ( github.com/oxtoacart/bpool v0.0.0-20190530202638-03653db5a59c github.com/patrickmn/go-cache v2.1.0+incompatible github.com/pkg/errors v0.9.1 + github.com/soheilhy/cmux v0.1.4 // indirect github.com/stretchr/testify v1.4.0 + github.com/technoweenie/multipartstreamer v1.0.1 // indirect + github.com/tmc/grpc-websocket-proxy v0.0.0-20200122045848-3419fae592fc // indirect + github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect go.etcd.io/bbolt v1.3.4 go.uber.org/zap v1.13.0 golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d golang.org/x/net v0.0.0-20200222125558-5a598a2470a0 + golang.org/x/time v0.0.0-20191024005414-555d28b269f0 // indirect google.golang.org/grpc v1.26.0 - gopkg.in/go-playground/validator.v9 v9.31.0 gopkg.in/src-d/go-git.v4 v4.13.1 gopkg.in/telegram-bot-api.v4 v4.6.4 + sigs.k8s.io/yaml v1.1.0 // indirect ) diff --git a/go.sum b/go.sum index 6011d507..63e5e123 100644 --- a/go.sum +++ b/go.sum @@ -64,9 +64,6 @@ github.com/bitly/go-simplejson v0.5.0/go.mod h1:cXHtHw4XUPsvGaxgjIAn8PhEWG9NfngE github.com/blang/semver v3.1.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= -github.com/boltdb/bolt v1.3.1 h1:JQmyP4ZBrce+ZQu0dY660FMfatumYDLun9hBCUVIkF4= -github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= -github.com/boombuler/barcode v1.0.1-0.20190219062509-6c824513bacc/go.mod h1:paBWMcWSl3LHKBqUq+rly7CNSldXjb2rDl3JlRe0mD8= github.com/bwmarrin/discordgo v0.20.2 h1:nA7jiTtqUA9lT93WL2jPjUp8ZTEInRujBdx1C9gkr20= github.com/bwmarrin/discordgo v0.20.2/go.mod h1:O9S4p+ofTFwB02em7jkpkV8M3R0/PUVOwN61zSZ0r4Q= github.com/cenkalti/backoff/v3 v3.0.0 h1:ske+9nBpD9qZsTBoF41nW5L+AIuFBKMeze18XQ3eG1c= @@ -75,14 +72,9 @@ github.com/census-instrumentation/opencensus-proto v0.2.0/go.mod h1:f6KPmirojxKA github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cheekybits/genny v1.0.0 h1:uGGa4nei+j20rOSeDeP5Of12XVm7TGUd4dJA9RDitfE= github.com/cheekybits/genny v1.0.0/go.mod h1:+tQajlRqAUrPI7DOSpB0XAqZYtQakVtB7wXkRAgjxjQ= -github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= -github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= -github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudflare/cloudflare-go v0.10.2 h1:VBodKICVPnwmDxstcW3biKcDSpFIfS/RELUXsZSBYK4= github.com/cloudflare/cloudflare-go v0.10.2/go.mod h1:qhVI5MKwBGhdNU89ZRz2plgYutcJ5PCekLxXn56w6SY= -github.com/cloudflare/cloudflare-go v0.10.9 h1:d8KOgLpYiC+Xq3T4tuO+/goM+RZvuO+T4pojuv8giL8= -github.com/cloudflare/cloudflare-go v0.10.9/go.mod h1:5TrsWH+3f4NV6WjtS5QFp+DifH81rph40gU374Sh0dQ= github.com/containerd/cgroups v0.0.0-20190919134610-bf292b21730f/go.mod h1:OApqhQ4XNSNC13gXIwDjhOQxjWa/NxkwZXJ1EvqT0ko= github.com/containerd/console v0.0.0-20180822173158-c12b1e7919c1/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw= github.com/containerd/containerd v1.3.0-beta.2.0.20190828155532-0293cbd26c69/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= @@ -94,6 +86,7 @@ github.com/containerd/fifo v0.0.0-20190226154929-a9fb20d87448/go.mod h1:ODA38xgv github.com/containerd/go-runc v0.0.0-20180907222934-5a6d9f37cfa3/go.mod h1:IV7qH3hrUgRmyYrtgEeGWJfWbgcHL9CSRruz2Vqcph0= github.com/containerd/ttrpc v0.0.0-20190828154514-0e0f228740de/go.mod h1:PvCDdDGpgqzQIzDW1TphrGLssLDZp2GuS+X5DkEJB8o= github.com/containerd/typeurl v0.0.0-20180627222232-a93fcdb778cd/go.mod h1:Cm3kwCdlkCfMSHURc+r6fwoGH6/F1hH3S4sg0rLFWPc= +github.com/coreos/bbolt v1.3.3 h1:n6AiVyVRKQFNb6mJlwESEvvLoDyiTzXX7ORAUlkeBdY= github.com/coreos/bbolt v1.3.3/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.18+incompatible h1:Zz1aXgDrFFi1nadh58tA9ktt06cmPTwNNP3dXwIq1lE= github.com/coreos/etcd v3.3.18+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= @@ -131,7 +124,6 @@ github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1 github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= github.com/ef-ds/deque v1.0.4-0.20190904040645-54cb57c252a1 h1:jFGzikHboUMRXmMBtwD/PbxoTHPs2919Irp/3rxMbvM= github.com/ef-ds/deque v1.0.4-0.20190904040645-54cb57c252a1/go.mod h1:HvODWzv6Y6kBf3Ah2WzN1bHjDUezGLaAhwuWVwfpEJs= -github.com/eknkc/basex v1.0.0/go.mod h1:k/F/exNEHFdbs3ZHuasoP2E7zeWwZblG84Y7Z59vQRo= github.com/emirpasic/gods v1.12.0 h1:QAUIPSaCu4G+POclxeqb3F+WPpdKqFGlw36+yOzGlrg= github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3ymbK86o= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= @@ -144,7 +136,6 @@ github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 h1:BHsljHzVlRcyQhjr github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= github.com/forestgiant/sliceutil v0.0.0-20160425183142-94783f95db6c h1:pBgVXWDXju1m8W4lnEeIqTHPOzhTUO81a7yknM/xQR4= github.com/forestgiant/sliceutil v0.0.0-20160425183142-94783f95db6c/go.mod h1:pFdJbAhRf7rh6YYMUdIQGyzne6zYL1tCUW8QV2B3UfY= -github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsouza/go-dockerclient v1.6.0 h1:f7j+AX94143JL1H3TiqSMkM4EcLDI0De1qD4GGn3Hig= @@ -162,10 +153,6 @@ github.com/go-ini/ini v1.44.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3I github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= -github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q= -github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= -github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD876Lmtgy7VtROAbHHXk8no= -github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-telegram-bot-api/telegram-bot-api v4.6.4+incompatible h1:2cauKuaELYAEARXRkq2LrJ0yDDv1rW7+wrTEdVL3uaU= github.com/go-telegram-bot-api/telegram-bot-api v4.6.4+incompatible/go.mod h1:qf9acutJ8cwBUhm1bqgz6Bei9/C/c93FPDljKWwsOgM= @@ -204,14 +191,12 @@ github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMyw github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.4.0 h1:xsAVV57WRhGj6kEIi8ReJzQlHHqcBYCElAvkovg3B/4= github.com/google/go-cmp v0.4.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= @@ -235,7 +220,6 @@ github.com/grpc-ecosystem/grpc-gateway v1.8.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t github.com/grpc-ecosystem/grpc-gateway v1.9.5 h1:UImYN5qQ8tuGpGE16ZmjvcTtTw24zw1QAp/SlnNrZhI= github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542/go.mod h1:Ow0tF8D4Kplbc8s8sSb3V2oUCygFHVp8gC3Dn6U4MNI= -github.com/hako/branca v0.0.0-20180808000428-10b799466ada/go.mod h1:tOPn4gvKEUWqIJNE+zpTeTALaRAXnrRqqSnPlO3VpEo= github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-multierror v0.0.0-20161216184304-ed905158d874/go.mod h1:JMRHfdO9jKNzS/+BTlxCjKNQHg/jZAft8U7LloJvN7I= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= @@ -283,26 +267,20 @@ github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/labbsr0x/bindman-dns-webhook v1.0.2/go.mod h1:p6b+VCXIR8NYKpDr8/dg1HKfQoRHCdcsROXKvmoehKA= github.com/labbsr0x/goh v1.0.1/go.mod h1:8K2UhVoaWXcCU7Lxoa2omWnC8gyW8px7/lmO61c027w= -github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y= -github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= github.com/lib/pq v1.3.0 h1:/qkRGz8zljWiDcFvgpwUpwIAPu3r07TDvs3Rws+o/pU= github.com/lib/pq v1.3.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/linode/linodego v0.10.0/go.mod h1:cziNP7pbvE3mXIPneHj0oRY8L1WtGEIKlZ8LANE4eXA= github.com/liquidweb/liquidweb-go v1.6.0/go.mod h1:UDcVnAMDkZxpw4Y7NOHkqoeiGacVLEIG/i5J9cyixzQ= github.com/lucas-clemente/quic-go v0.14.1 h1:c1aKoBZKOPA+49q96B1wGkibyPP0AxYh45WuAoq+87E= github.com/lucas-clemente/quic-go v0.14.1/go.mod h1:Vn3/Fb0/77b02SGhQk36KzOUmXgVpFfizUfW5WMaqyU= -github.com/mailru/easyjson v0.0.0-20180730094502-03f2033d19d5/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/marten-seemann/chacha20 v0.2.0 h1:f40vqzzx+3GdOmzQoItkLX5WLvHgPgyYqFFIO5Gh4hQ= github.com/marten-seemann/chacha20 v0.2.0/go.mod h1:HSdjFau7GzYRj+ahFNwsO3ouVJr1HFkWoEwNDb4TMtE= github.com/marten-seemann/qpack v0.1.0/go.mod h1:LFt1NU/Ptjip0C2CPkhimBz5CGE3WGDAUWqna+CNTrI= github.com/marten-seemann/qtls v0.4.1 h1:YlT8QP3WCCvvok7MGEZkMldXbyqgr8oFg5/n8Gtbkks= github.com/marten-seemann/qtls v0.4.1/go.mod h1:pxVXcHHw1pNIt8Qo0pwSYQEoZ8yYOOPXTCZLQQunvRc= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= -github.com/mattn/go-runewidth v0.0.6/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= -github.com/mattn/go-runewidth v0.0.7/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-tty v0.0.0-20180219170247-931426f7535a/go.mod h1:XPvLUNfbS4fJH25nqRHfWLMa1ONC8Amw+mIA639KxkE= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= @@ -310,12 +288,8 @@ github.com/mholt/certmagic v0.9.3 h1:RmzuNJ5mpFplDbyS41z+gGgE/py24IX6m0nHZ0yNTQU github.com/mholt/certmagic v0.9.3/go.mod h1:nu8jbsbtwK4205EDH/ZUMTKsfYpJA1Q7MKXHfgTihNw= github.com/micro/cli/v2 v2.1.2 h1:43J1lChg/rZCC1rvdqZNFSQDrGT7qfMrtp6/ztpIkEM= github.com/micro/cli/v2 v2.1.2/go.mod h1:EguNh6DAoWKm9nmk+k/Rg0H3lQnDxqzu5x5srOtGtYg= -github.com/micro/go-micro v1.18.0 h1:gP70EZVHpJuUIT0YWth192JmlIci+qMOEByHm83XE9E= -github.com/micro/go-micro/v2 v2.3.1-0.20200331090613-76ade7efd9b8/go.mod h1:lYuHYFPjY3QE9fdiy3F2awXcsXTdB68AwoY3RQ3dPN4= github.com/micro/mdns v0.3.0 h1:bYycYe+98AXR3s8Nq5qvt6C573uFTDPIYzJemWON0QE= github.com/micro/mdns v0.3.0/go.mod h1:KJ0dW7KmicXU2BV++qkLlmHYcVv7/hHnbtguSWt9Aoc= -github.com/micro/micro/v2 v2.4.0 h1:GlbLaD/50KaSFym7GCQZ/2I4fuTTX9U4Zftni4ImJ40= -github.com/micro/micro/v2 v2.4.0/go.mod h1:/7lxBaU/Isx3USObggNVw6x6pdIJzTDexee7EsARD+A= github.com/miekg/dns v1.1.3/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.15/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.27 h1:aEH/kqUzUxGJ/UHcEKdJY+ugH6WEzsEBBSPa8zuy1aM= @@ -349,7 +323,6 @@ github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxzi github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32/go.mod h1:9wM+0iRr9ahx58uYLpLIr5fm8diHn0JbqRycJi6w0Ms= -github.com/netdata/go-orchestrator v0.0.0-20190905093727-c793edba0e8f/go.mod h1:ECF8anFVCt/TfTIWVPgPrNaYJXtAtpAOF62ugDbw41A= github.com/nlopes/slack v0.6.1-0.20191106133607-d06c2a2b3249 h1:Pr5gZa2VcmktVwq0lyC39MsN5tz356vC/pQHKvq+QBo= github.com/nlopes/slack v0.6.1-0.20191106133607-d06c2a2b3249/go.mod h1:JzQ9m3PMAqcpeCam7UaHSuBuupz7CmpjehYMayT6YOk= github.com/nrdcg/auroradns v1.0.0/go.mod h1:6JPXKzIRzZzMqtTDgueIhTi6rFf1QvYE/HzqidhOhjw= @@ -357,8 +330,6 @@ github.com/nrdcg/dnspod-go v0.3.0/go.mod h1:vZSoFSFeQVm2gWLMkyX61LZ8HI3BaqtHZWgP github.com/nrdcg/goinwx v0.6.1/go.mod h1:XPiut7enlbEdntAqalBIqcYcTEVhpv/dKWgDCX2SwKQ= github.com/nrdcg/namesilo v0.2.1/go.mod h1:lwMvfQTyYq+BbjJd30ylEG4GPSS6PII0Tia4rRpRiyw= github.com/olekukonko/tablewriter v0.0.1/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= -github.com/olekukonko/tablewriter v0.0.3/go.mod h1:YZeBtGzYYEsCHp2LST/u/0NDwGkRoBtmn1cIWCJiS6M= -github.com/olekukonko/tablewriter v0.0.4/go.mod h1:zq6QwlOf5SlnkVbMSr5EoBv3636FWnp+qbPhuoO21uA= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= @@ -382,7 +353,6 @@ github.com/oxtoacart/bpool v0.0.0-20190530202638-03653db5a59c h1:rp5dCmg/yLR3mgF github.com/oxtoacart/bpool v0.0.0-20190530202638-03653db5a59c/go.mod h1:X07ZCGwUbLaax7L0S3Tw4hpejzu63ZrrQiUe6W0hcy0= github.com/patrickmn/go-cache v2.1.0+incompatible h1:HRMgzkcYKYpi3C8ajMPV8OFXaaRUnok+kx1WdO15EQc= github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= -github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-buffruneio v0.2.0/go.mod h1:JkE26KsDizTr40EUHkXVtNPvgGtbSNq5BcowyYOWdKo= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -391,7 +361,6 @@ github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= github.com/pkg/errors v0.9.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= -github.com/pquerna/otp v1.2.0/go.mod h1:dkJfzwRKNiegxyNb54X/3fLwhCynbMspSyWKnvi1AEg= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= @@ -420,7 +389,6 @@ github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sacloud/libsacloud v1.26.1/go.mod h1:79ZwATmHLIFZIMd7sxA3LwzVy/B77uj3LDoToVTxDoQ= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= -github.com/serenize/snaker v0.0.0-20171204205717-a683aaf2d516/go.mod h1:Yow6lPLSAXx2ifx470yD/nUe22Dv5vBvxK/UK9UUTVs= github.com/sergi/go-diff v1.0.0 h1:Kpca3qRNrduNnOQeazBd0ysaKrUJiIuISHxogkT9RPQ= github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= @@ -431,7 +399,6 @@ github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4 github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/skratchdot/open-golang v0.0.0-20160302144031-75fb7ed4208c/go.mod h1:sUM3LWHvSMaG192sy56D9F7CNvL7jUJVXoqM1QKLnog= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= -github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM= github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/soheilhy/cmux v0.1.4 h1:0HKaf1o97UwFjHH9o5XsHUOF+tqmdA7KEzXLpiyaw0E= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= @@ -454,7 +421,6 @@ github.com/transip/gotransip v0.0.0-20190812104329-6d8d9179b66f/go.mod h1:i0f4R4 github.com/uber-go/atomic v1.3.2/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex1PFV8g= github.com/urfave/cli v0.0.0-20171014202726-7bc6a0acffa5/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= -github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/vultr/govultr v0.1.4/go.mod h1:9H008Uxr/C4vFNGLqKx232C206GL0PBHzOP0809bGNA= github.com/xanzy/ssh-agent v0.2.1 h1:TCbipTQL2JiiCprBWx9frJ2eJlCYT00NmctrHxVAr70= github.com/xanzy/ssh-agent v0.2.1/go.mod h1:mLlQY/MoOhWBj+gOGMQkOeiEvkx+8pJSI+0Bx9h2kr4= @@ -464,8 +430,6 @@ github.com/xeipuuv/gojsonschema v0.0.0-20180618132009-1d523034197f/go.mod h1:5yf github.com/xeipuuv/gojsonschema v1.1.0/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5Qo6v2eYzo7kUS51QINcR5jNpbZS8= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= -github.com/xlab/treeprint v0.0.0-20181112141820-a009c3971eca/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg= -go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= go.etcd.io/bbolt v1.3.4 h1:hi1bXHMVrlQh6WwxAy+qZCV/SYIlqo+Ushwdpa4tAKg= go.etcd.io/bbolt v1.3.4/go.mod h1:G5EMThwa9y8QZGBClrRx5EY+Yw9kAhnjy3bSjsnlVTQ= go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= @@ -543,14 +507,11 @@ golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLL golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190930134127-c5a3c61f89f3/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191027093000-83d349e8ac1a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20191126235420-ef20fe5d7933/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200222125558-5a598a2470a0 h1:MsuvTghUPjX762sGLnGsxC3HM0B5r83wEtYcYR8/vRs= golang.org/x/net v0.0.0-20200222125558-5a598a2470a0/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= -golang.org/x/net v0.0.0-20200324143707-d3edc9973b7e h1:3G+cUijn7XD+S4eJFddp53Pv7+slrESplyjG25HgL+k= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -569,7 +530,6 @@ golang.org/x/sys v0.0.0-20190129075346-302c3dd5f1cc/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190209173611-3b5209105503/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190221075227-b4e8571b14e0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190403152447-81d4e9dc473e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -585,7 +545,6 @@ golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe h1:6fAMxZRR6sl1Uq8U61gxU+kPTs2tR8uOySCbBP7BN/M= golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20191110163157-d32e6e3b99c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5 h1:LfCXLvNmTYH9kEmVgqbnsWfruoXZIrh4YBgqVHtDvw0= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -658,23 +617,16 @@ google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyac google.golang.org/grpc v1.26.0 h1:2dTRdpdFEEhJYQD8EMLB61nnrzSCTbG38PhqdhvOltg= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= -gopkg.in/alexcesaro/quotedprintable.v3 v3.0.0-20150716171945-2caba252f4dc/go.mod h1:m7x9LTH6d71AHyAX77c9yqWCCa3UKHcVEj9y7hAtKDk= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= -gopkg.in/go-playground/assert.v1 v1.2.1 h1:xoYuJVE7KT85PYWrN730RguIQO0ePzVRfFMXadIrXTM= -gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= -gopkg.in/go-playground/validator.v9 v9.31.0 h1:bmXmP2RSNtFES+bn4uYuHT7iJFJv7Vj+an+ZQdDaD1M= -gopkg.in/go-playground/validator.v9 v9.31.0/go.mod h1:+c9/zcJMFNgbLvly1L1V+PpxWdVbfP1avr/N00E2vyQ= -gopkg.in/gomail.v2 v2.0.0-20160411212932-81ebce5c23df/go.mod h1:LRQQ+SO6ZHR7tOkpBDuZnXENFzX8qRjMDMyPD6BRkCw= gopkg.in/h2non/gock.v1 v1.0.15/go.mod h1:sX4zAkdYX1TRGJ2JY156cFspQn4yRWn6p9EMdODlynE= gopkg.in/ini.v1 v1.42.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.44.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ns1/ns1-go.v2 v2.0.0-20190730140822-b51389932cbc/go.mod h1:VV+3haRsgDiVLxyifmMBrBIuCWFBPYKbRssXB9z67Hw= -gopkg.in/olivere/elastic.v5 v5.0.83/go.mod h1:LXF6q9XNBxpMqrcgax95C6xyARXWbbCXUrtTxrNrxJI= gopkg.in/resty.v1 v1.9.1/go.mod h1:vo52Hzryw9PnPHcJfPsBiFW62XhNx5OczbV9y+IMpgc= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= gopkg.in/square/go-jose.v2 v2.3.1 h1:SK5KegNXmKmqE342YYN2qPHEnUYeoMiXXl1poUlI+o4= From 57b758db7ee8431e4af71a10423bcd76257fe24a Mon Sep 17 00:00:00 2001 From: Asim Aslam Date: Fri, 10 Apr 2020 22:09:06 +0100 Subject: [PATCH 14/44] push --- README.md | 1 - 1 file changed, 1 deletion(-) diff --git a/README.md b/README.md index 73837149..d6294ace 100644 --- a/README.md +++ b/README.md @@ -45,4 +45,3 @@ are pluggable and allows Go Micro to be runtime agnostic. You can plugin any und ## Getting Started See the [docs](https://micro.mu/docs/framework.html) for detailed information on the architecture, installation and use of go-micro. - From b979db6d9dbbab972dd3f41ad8b50125443355d0 Mon Sep 17 00:00:00 2001 From: Asim Aslam Date: Fri, 10 Apr 2020 23:29:15 +0100 Subject: [PATCH 15/44] remove sync event --- sync/event/event.go | 27 --------------------------- 1 file changed, 27 deletions(-) delete mode 100644 sync/event/event.go diff --git a/sync/event/event.go b/sync/event/event.go deleted file mode 100644 index bd85d16c..00000000 --- a/sync/event/event.go +++ /dev/null @@ -1,27 +0,0 @@ -// Package event provides a distributed log interface -package event - -// Event provides a distributed log interface -type Event interface { - // Log retrieves the log with an id/name - Log(id string) (Log, error) -} - -// Log is an individual event log -type Log interface { - // Close the log handle - Close() error - // Log ID - Id() string - // Read will read the next record - Read() (*Record, error) - // Go to an offset - Seek(offset int64) error - // Write an event to the log - Write(*Record) error -} - -type Record struct { - Metadata map[string]interface{} - Data []byte -} From bc71640fd942f1e27e58e68997dc001b7b53a549 Mon Sep 17 00:00:00 2001 From: Vasiliy Tolstov Date: Sat, 11 Apr 2020 03:46:54 +0300 Subject: [PATCH 16/44] broker: swap default broker from eats to http (#1524) * broker: swap default broker from eats to http Signed-off-by: Vasiliy Tolstov --- broker/default.go | 504 ----------------------------- broker/http.go | 709 +++++++++++++++++++++++++++++++++++++++++ broker/http/http.go | 11 + broker/http/options.go | 23 ++ broker/http_test.go | 384 ++++++++++++++++++++++ config/cmd/cmd.go | 2 + go.mod | 2 - go.sum | 12 - 8 files changed, 1129 insertions(+), 518 deletions(-) delete mode 100644 broker/default.go create mode 100644 broker/http.go create mode 100644 broker/http/http.go create mode 100644 broker/http/options.go create mode 100644 broker/http_test.go diff --git a/broker/default.go b/broker/default.go deleted file mode 100644 index ae6562ce..00000000 --- a/broker/default.go +++ /dev/null @@ -1,504 +0,0 @@ -package broker - -import ( - "context" - "errors" - "net" - "net/url" - "strconv" - "strings" - "sync" - "time" - - "github.com/micro/go-micro/v2/codec/json" - "github.com/micro/go-micro/v2/logger" - "github.com/micro/go-micro/v2/registry" - "github.com/micro/go-micro/v2/util/addr" - "github.com/nats-io/nats-server/v2/server" - nats "github.com/nats-io/nats.go" -) - -type natsBroker struct { - sync.Once - sync.RWMutex - - // indicate if we're connected - connected bool - - // address to bind routes to - addrs []string - // servers for the client - servers []string - - // client connection and nats opts - conn *nats.Conn - opts Options - nopts nats.Options - - // should we drain the connection - drain bool - closeCh chan (error) - - // embedded server - server *server.Server - // configure to use local server - local bool - // server exit channel - exit chan bool -} - -type subscriber struct { - s *nats.Subscription - opts SubscribeOptions -} - -type publication struct { - t string - err error - m *Message -} - -func (p *publication) Topic() string { - return p.t -} - -func (p *publication) Message() *Message { - return p.m -} - -func (p *publication) Ack() error { - // nats does not support acking - return nil -} - -func (p *publication) Error() error { - return p.err -} - -func (s *subscriber) Options() SubscribeOptions { - return s.opts -} - -func (s *subscriber) Topic() string { - return s.s.Subject -} - -func (s *subscriber) Unsubscribe() error { - return s.s.Unsubscribe() -} - -func (n *natsBroker) Address() string { - n.RLock() - defer n.RUnlock() - - if n.server != nil { - return n.server.ClusterAddr().String() - } - - if n.conn != nil && n.conn.IsConnected() { - return n.conn.ConnectedUrl() - } - - if len(n.addrs) > 0 { - return n.addrs[0] - } - - return "127.0.0.1:-1" -} - -func (n *natsBroker) setAddrs(addrs []string) []string { - //nolint:prealloc - var cAddrs []string - for _, addr := range addrs { - if len(addr) == 0 { - continue - } - if !strings.HasPrefix(addr, "nats://") { - addr = "nats://" + addr - } - cAddrs = append(cAddrs, addr) - } - // if there's no address and we weren't told to - // embed a local server then use the default url - if len(cAddrs) == 0 && !n.local { - cAddrs = []string{nats.DefaultURL} - } - return cAddrs -} - -// serve stats a local nats server if needed -func (n *natsBroker) serve(exit chan bool) error { - // local server address - host := "127.0.0.1" - port := -1 - - // cluster address - caddr := "0.0.0.0" - cport := -1 - - // with no address we just default it - // this is a local client address - if len(n.addrs) > 0 { - address := n.addrs[0] - if strings.HasPrefix(address, "nats://") { - address = strings.TrimPrefix(address, "nats://") - } - - // parse out the address - h, p, err := net.SplitHostPort(address) - if err == nil { - caddr = h - cport, _ = strconv.Atoi(p) - } - } - - // 1. create new server - // 2. register the server - // 3. connect to other servers - - // set cluster opts - cOpts := server.ClusterOpts{ - Host: caddr, - Port: cport, - } - - // get the routes for other nodes - var routes []*url.URL - - // get existing nats servers to connect to - services, err := n.opts.Registry.GetService("go.micro.nats.broker") - if err == nil { - for _, service := range services { - for _, node := range service.Nodes { - u, err := url.Parse("nats://" + node.Address) - if err != nil { - if logger.V(logger.InfoLevel, logger.DefaultLogger) { - logger.Info(err) - } - continue - } - // append to the cluster routes - routes = append(routes, u) - } - } - } - - // try get existing server - s := n.server - - if s != nil { - // stop the existing server - s.Shutdown() - } - - s, err = server.NewServer(&server.Options{ - // Specify the host - Host: host, - // Use a random port - Port: port, - // Set the cluster ops - Cluster: cOpts, - // Set the routes - Routes: routes, - NoLog: true, - NoSigs: true, - MaxControlLine: 2048, - TLSConfig: n.opts.TLSConfig, - }) - if err != nil { - return err - } - - // save the server - n.server = s - - // start the server - go s.Start() - - var ready bool - - // wait till its ready for connections - for i := 0; i < 3; i++ { - if s.ReadyForConnections(time.Second) { - ready = true - break - } - } - - if !ready { - return errors.New("server not ready") - } - - // set the client address - n.servers = []string{s.ClientURL()} - - go func() { - var advertise string - - // parse out the address - _, port, err := net.SplitHostPort(s.ClusterAddr().String()) - if err == nil { - addr, _ := addr.Extract("") - advertise = net.JoinHostPort(addr, port) - } else { - s.ClusterAddr().String() - } - - // register the cluster address - for { - select { - case err := <-n.closeCh: - if err != nil { - if logger.V(logger.InfoLevel, logger.DefaultLogger) { - logger.Info(err) - } - } - case <-exit: - // deregister on exit - n.opts.Registry.Deregister(®istry.Service{ - Name: "go.micro.nats.broker", - Version: "v2", - Nodes: []*registry.Node{ - {Id: s.ID(), Address: advertise}, - }, - }) - s.Shutdown() - return - default: - // register the broker - n.opts.Registry.Register(®istry.Service{ - Name: "go.micro.nats.broker", - Version: "v2", - Nodes: []*registry.Node{ - {Id: s.ID(), Address: advertise}, - }, - }, registry.RegisterTTL(time.Minute)) - time.Sleep(time.Minute) - } - } - }() - - return nil -} - -func (n *natsBroker) Connect() error { - n.Lock() - defer n.Unlock() - - if !n.connected { - // create exit chan - n.exit = make(chan bool) - - // start the local server - if err := n.serve(n.exit); err != nil { - return err - } - - // set to connected - } - - status := nats.CLOSED - if n.conn != nil { - status = n.conn.Status() - } - - switch status { - case nats.CONNECTED, nats.RECONNECTING, nats.CONNECTING: - return nil - default: // DISCONNECTED or CLOSED or DRAINING - opts := n.nopts - opts.DrainTimeout = 1 * time.Second - opts.AsyncErrorCB = n.onAsyncError - opts.DisconnectedErrCB = n.onDisconnectedError - opts.ClosedCB = n.onClose - opts.Servers = n.servers - opts.Secure = n.opts.Secure - opts.TLSConfig = n.opts.TLSConfig - - // secure might not be set - if n.opts.TLSConfig != nil { - opts.Secure = true - } - - c, err := opts.Connect() - if err != nil { - return err - } - n.conn = c - - n.connected = true - - return nil - } -} - -func (n *natsBroker) Disconnect() error { - n.RLock() - defer n.RUnlock() - - if !n.connected { - return nil - } - - // drain the connection if specified - if n.drain { - n.conn.Drain() - } - - // close the client connection - n.conn.Close() - - // shutdown the local server - // and deregister - if n.server != nil { - select { - case <-n.exit: - default: - close(n.exit) - } - } - - // set not connected - n.connected = false - - return nil -} - -func (n *natsBroker) Init(opts ...Option) error { - n.setOption(opts...) - return nil -} - -func (n *natsBroker) Options() Options { - return n.opts -} - -func (n *natsBroker) Publish(topic string, msg *Message, opts ...PublishOption) error { - b, err := n.opts.Codec.Marshal(msg) - if err != nil { - return err - } - n.RLock() - defer n.RUnlock() - return n.conn.Publish(topic, b) -} - -func (n *natsBroker) Subscribe(topic string, handler Handler, opts ...SubscribeOption) (Subscriber, error) { - if n.conn == nil { - return nil, errors.New("not connected") - } - - opt := SubscribeOptions{ - AutoAck: true, - Context: context.Background(), - } - - for _, o := range opts { - o(&opt) - } - - fn := func(msg *nats.Msg) { - var m Message - pub := &publication{t: msg.Subject} - eh := n.opts.ErrorHandler - err := n.opts.Codec.Unmarshal(msg.Data, &m) - pub.err = err - pub.m = &m - if err != nil { - m.Body = msg.Data - if logger.V(logger.ErrorLevel, logger.DefaultLogger) { - logger.Error(err) - } - if eh != nil { - eh(pub) - } - return - } - if err := handler(pub); err != nil { - pub.err = err - if logger.V(logger.ErrorLevel, logger.DefaultLogger) { - logger.Error(err) - } - if eh != nil { - eh(pub) - } - } - } - - var sub *nats.Subscription - var err error - - n.RLock() - if len(opt.Queue) > 0 { - sub, err = n.conn.QueueSubscribe(topic, opt.Queue, fn) - } else { - sub, err = n.conn.Subscribe(topic, fn) - } - n.RUnlock() - if err != nil { - return nil, err - } - return &subscriber{s: sub, opts: opt}, nil -} - -func (n *natsBroker) String() string { - return "eats" -} - -func (n *natsBroker) setOption(opts ...Option) { - for _, o := range opts { - o(&n.opts) - } - - n.Once.Do(func() { - n.nopts = nats.GetDefaultOptions() - }) - - // local embedded server - n.local = true - // set to drain - n.drain = true - - if !n.opts.Secure { - n.opts.Secure = n.nopts.Secure - } - - if n.opts.TLSConfig == nil { - n.opts.TLSConfig = n.nopts.TLSConfig - } - - n.addrs = n.setAddrs(n.opts.Addrs) -} - -func (n *natsBroker) onClose(conn *nats.Conn) { - n.closeCh <- nil -} - -func (n *natsBroker) onDisconnectedError(conn *nats.Conn, err error) { - n.closeCh <- err -} - -func (n *natsBroker) onAsyncError(conn *nats.Conn, sub *nats.Subscription, err error) { - // There are kinds of different async error nats might callback, but we are interested - // in ErrDrainTimeout only here. - if err == nats.ErrDrainTimeout { - n.closeCh <- err - } -} - -func NewBroker(opts ...Option) Broker { - options := Options{ - // Default codec - Codec: json.Marshaler{}, - Context: context.Background(), - Registry: registry.DefaultRegistry, - } - - n := &natsBroker{ - opts: options, - closeCh: make(chan error), - } - n.setOption(opts...) - - return n -} diff --git a/broker/http.go b/broker/http.go new file mode 100644 index 00000000..a4a90698 --- /dev/null +++ b/broker/http.go @@ -0,0 +1,709 @@ +// Package http provides a http based message broker +package broker + +import ( + "bytes" + "context" + "crypto/tls" + "errors" + "fmt" + "io" + "io/ioutil" + "math/rand" + "net" + "net/http" + "net/url" + "runtime" + "sync" + "time" + + "github.com/google/uuid" + "github.com/micro/go-micro/v2/codec/json" + merr "github.com/micro/go-micro/v2/errors" + "github.com/micro/go-micro/v2/registry" + "github.com/micro/go-micro/v2/registry/cache" + maddr "github.com/micro/go-micro/v2/util/addr" + mnet "github.com/micro/go-micro/v2/util/net" + mls "github.com/micro/go-micro/v2/util/tls" + "golang.org/x/net/http2" +) + +// HTTP Broker is a point to point async broker +type httpBroker struct { + id string + address string + opts Options + + mux *http.ServeMux + + c *http.Client + r registry.Registry + + sync.RWMutex + subscribers map[string][]*httpSubscriber + running bool + exit chan chan error + + // offline message inbox + mtx sync.RWMutex + inbox map[string][][]byte +} + +type httpSubscriber struct { + opts SubscribeOptions + id string + topic string + fn Handler + svc *registry.Service + hb *httpBroker +} + +type httpEvent struct { + m *Message + t string + err error +} + +var ( + DefaultSubPath = "/" + serviceName = "micro.http.broker" + broadcastVersion = "ff.http.broadcast" + registerTTL = time.Minute + registerInterval = time.Second * 30 +) + +func init() { + rand.Seed(time.Now().Unix()) +} + +func newTransport(config *tls.Config) *http.Transport { + if config == nil { + config = &tls.Config{ + InsecureSkipVerify: true, + } + } + + dialTLS := func(network string, addr string) (net.Conn, error) { + return tls.Dial(network, addr, config) + } + + t := &http.Transport{ + Proxy: http.ProxyFromEnvironment, + Dial: (&net.Dialer{ + Timeout: 30 * time.Second, + KeepAlive: 30 * time.Second, + }).Dial, + TLSHandshakeTimeout: 10 * time.Second, + DialTLS: dialTLS, + } + runtime.SetFinalizer(&t, func(tr **http.Transport) { + (*tr).CloseIdleConnections() + }) + + // setup http2 + http2.ConfigureTransport(t) + + return t +} + +func newHttpBroker(opts ...Option) Broker { + options := Options{ + Codec: json.Marshaler{}, + Context: context.TODO(), + Registry: registry.DefaultRegistry, + } + + for _, o := range opts { + o(&options) + } + + // set address + addr := ":0" + if len(options.Addrs) > 0 && len(options.Addrs[0]) > 0 { + addr = options.Addrs[0] + } + + h := &httpBroker{ + id: uuid.New().String(), + address: addr, + opts: options, + r: options.Registry, + c: &http.Client{Transport: newTransport(options.TLSConfig)}, + subscribers: make(map[string][]*httpSubscriber), + exit: make(chan chan error), + mux: http.NewServeMux(), + inbox: make(map[string][][]byte), + } + + // specify the message handler + h.mux.Handle(DefaultSubPath, h) + + // get optional handlers + if h.opts.Context != nil { + handlers, ok := h.opts.Context.Value("http_handlers").(map[string]http.Handler) + if ok { + for pattern, handler := range handlers { + h.mux.Handle(pattern, handler) + } + } + } + + return h +} + +func (h *httpEvent) Ack() error { + return nil +} + +func (h *httpEvent) Error() error { + return h.err +} + +func (h *httpEvent) Message() *Message { + return h.m +} + +func (h *httpEvent) Topic() string { + return h.t +} + +func (h *httpSubscriber) Options() SubscribeOptions { + return h.opts +} + +func (h *httpSubscriber) Topic() string { + return h.topic +} + +func (h *httpSubscriber) Unsubscribe() error { + return h.hb.unsubscribe(h) +} + +func (h *httpBroker) saveMessage(topic string, msg []byte) { + h.mtx.Lock() + defer h.mtx.Unlock() + + // get messages + c := h.inbox[topic] + + // save message + c = append(c, msg) + + // max length 64 + if len(c) > 64 { + c = c[:64] + } + + // save inbox + h.inbox[topic] = c +} + +func (h *httpBroker) getMessage(topic string, num int) [][]byte { + h.mtx.Lock() + defer h.mtx.Unlock() + + // get messages + c, ok := h.inbox[topic] + if !ok { + return nil + } + + // more message than requests + if len(c) >= num { + msg := c[:num] + h.inbox[topic] = c[num:] + return msg + } + + // reset inbox + h.inbox[topic] = nil + + // return all messages + return c +} + +func (h *httpBroker) subscribe(s *httpSubscriber) error { + h.Lock() + defer h.Unlock() + + if err := h.r.Register(s.svc, registry.RegisterTTL(registerTTL)); err != nil { + return err + } + + h.subscribers[s.topic] = append(h.subscribers[s.topic], s) + return nil +} + +func (h *httpBroker) unsubscribe(s *httpSubscriber) error { + h.Lock() + defer h.Unlock() + + //nolint:prealloc + var subscribers []*httpSubscriber + + // look for subscriber + for _, sub := range h.subscribers[s.topic] { + // deregister and skip forward + if sub == s { + _ = h.r.Deregister(sub.svc) + continue + } + // keep subscriber + subscribers = append(subscribers, sub) + } + + // set subscribers + h.subscribers[s.topic] = subscribers + + return nil +} + +func (h *httpBroker) run(l net.Listener) { + t := time.NewTicker(registerInterval) + defer t.Stop() + + for { + select { + // heartbeat for each subscriber + case <-t.C: + h.RLock() + for _, subs := range h.subscribers { + for _, sub := range subs { + _ = h.r.Register(sub.svc, registry.RegisterTTL(registerTTL)) + } + } + h.RUnlock() + // received exit signal + case ch := <-h.exit: + ch <- l.Close() + h.RLock() + for _, subs := range h.subscribers { + for _, sub := range subs { + _ = h.r.Deregister(sub.svc) + } + } + h.RUnlock() + return + } + } +} + +func (h *httpBroker) ServeHTTP(w http.ResponseWriter, req *http.Request) { + if req.Method != "POST" { + err := merr.BadRequest("go.micro.broker", "Method not allowed") + http.Error(w, err.Error(), http.StatusMethodNotAllowed) + return + } + defer req.Body.Close() + + req.ParseForm() + + b, err := ioutil.ReadAll(req.Body) + if err != nil { + errr := merr.InternalServerError("go.micro.broker", "Error reading request body: %v", err) + w.WriteHeader(500) + w.Write([]byte(errr.Error())) + return + } + + var m *Message + if err = h.opts.Codec.Unmarshal(b, &m); err != nil { + errr := merr.InternalServerError("go.micro.broker", "Error parsing request body: %v", err) + w.WriteHeader(500) + w.Write([]byte(errr.Error())) + return + } + + topic := m.Header["Micro-Topic"] + //delete(m.Header, ":topic") + + if len(topic) == 0 { + errr := merr.InternalServerError("go.micro.broker", "Topic not found") + w.WriteHeader(500) + w.Write([]byte(errr.Error())) + return + } + + p := &httpEvent{m: m, t: topic} + id := req.Form.Get("id") + + //nolint:prealloc + var subs []Handler + + h.RLock() + for _, subscriber := range h.subscribers[topic] { + if id != subscriber.id { + continue + } + subs = append(subs, subscriber.fn) + } + h.RUnlock() + + // execute the handler + for _, fn := range subs { + p.err = fn(p) + } +} + +func (h *httpBroker) Address() string { + h.RLock() + defer h.RUnlock() + return h.address +} + +func (h *httpBroker) Connect() error { + h.RLock() + if h.running { + h.RUnlock() + return nil + } + h.RUnlock() + + h.Lock() + defer h.Unlock() + + var l net.Listener + var err error + + if h.opts.Secure || h.opts.TLSConfig != nil { + config := h.opts.TLSConfig + + fn := func(addr string) (net.Listener, error) { + if config == nil { + hosts := []string{addr} + + // check if its a valid host:port + if host, _, err := net.SplitHostPort(addr); err == nil { + if len(host) == 0 { + hosts = maddr.IPs() + } else { + hosts = []string{host} + } + } + + // generate a certificate + cert, err := mls.Certificate(hosts...) + if err != nil { + return nil, err + } + config = &tls.Config{Certificates: []tls.Certificate{cert}} + } + return tls.Listen("tcp", addr, config) + } + + l, err = mnet.Listen(h.address, fn) + } else { + fn := func(addr string) (net.Listener, error) { + return net.Listen("tcp", addr) + } + + l, err = mnet.Listen(h.address, fn) + } + + if err != nil { + return err + } + + addr := h.address + h.address = l.Addr().String() + + go http.Serve(l, h.mux) + go func() { + h.run(l) + h.Lock() + h.opts.Addrs = []string{addr} + h.address = addr + h.Unlock() + }() + + // get registry + reg := h.opts.Registry + if reg == nil { + reg = registry.DefaultRegistry + } + // set cache + h.r = cache.New(reg) + + // set running + h.running = true + return nil +} + +func (h *httpBroker) Disconnect() error { + h.RLock() + if !h.running { + h.RUnlock() + return nil + } + h.RUnlock() + + h.Lock() + defer h.Unlock() + + // stop cache + rc, ok := h.r.(cache.Cache) + if ok { + rc.Stop() + } + + // exit and return err + ch := make(chan error) + h.exit <- ch + err := <-ch + + // set not running + h.running = false + return err +} + +func (h *httpBroker) Init(opts ...Option) error { + h.RLock() + if h.running { + h.RUnlock() + return errors.New("cannot init while connected") + } + h.RUnlock() + + h.Lock() + defer h.Unlock() + + for _, o := range opts { + o(&h.opts) + } + + if len(h.opts.Addrs) > 0 && len(h.opts.Addrs[0]) > 0 { + h.address = h.opts.Addrs[0] + } + + if len(h.id) == 0 { + h.id = "go.micro.http.broker-" + uuid.New().String() + } + + // get registry + reg := h.opts.Registry + if reg == nil { + reg = registry.DefaultRegistry + } + + // get cache + if rc, ok := h.r.(cache.Cache); ok { + rc.Stop() + } + + // set registry + h.r = cache.New(reg) + + // reconfigure tls config + if c := h.opts.TLSConfig; c != nil { + h.c = &http.Client{ + Transport: newTransport(c), + } + } + + return nil +} + +func (h *httpBroker) Options() Options { + return h.opts +} + +func (h *httpBroker) Publish(topic string, msg *Message, opts ...PublishOption) error { + // create the message first + m := &Message{ + Header: make(map[string]string), + Body: msg.Body, + } + + for k, v := range msg.Header { + m.Header[k] = v + } + + m.Header["Micro-Topic"] = topic + + // encode the message + b, err := h.opts.Codec.Marshal(m) + if err != nil { + return err + } + + // save the message + h.saveMessage(topic, b) + + // now attempt to get the service + h.RLock() + s, err := h.r.GetService(serviceName) + if err != nil { + h.RUnlock() + return err + } + h.RUnlock() + + pub := func(node *registry.Node, t string, b []byte) error { + scheme := "http" + + // check if secure is added in metadata + if node.Metadata["secure"] == "true" { + scheme = "https" + } + + vals := url.Values{} + vals.Add("id", node.Id) + + uri := fmt.Sprintf("%s://%s%s?%s", scheme, node.Address, DefaultSubPath, vals.Encode()) + r, err := h.c.Post(uri, "application/json", bytes.NewReader(b)) + if err != nil { + return err + } + + // discard response body + io.Copy(ioutil.Discard, r.Body) + r.Body.Close() + return nil + } + + srv := func(s []*registry.Service, b []byte) { + for _, service := range s { + var nodes []*registry.Node + + for _, node := range service.Nodes { + // only use nodes tagged with broker http + if node.Metadata["broker"] != "http" { + continue + } + + // look for nodes for the topic + if node.Metadata["topic"] != topic { + continue + } + + nodes = append(nodes, node) + } + + // only process if we have nodes + if len(nodes) == 0 { + continue + } + + switch service.Version { + // broadcast version means broadcast to all nodes + case broadcastVersion: + var success bool + + // publish to all nodes + for _, node := range nodes { + // publish async + if err := pub(node, topic, b); err == nil { + success = true + } + } + + // save if it failed to publish at least once + if !success { + h.saveMessage(topic, b) + } + default: + // select node to publish to + node := nodes[rand.Int()%len(nodes)] + + // publish async to one node + if err := pub(node, topic, b); err != nil { + // if failed save it + h.saveMessage(topic, b) + } + } + } + } + + // do the rest async + go func() { + // get a third of the backlog + messages := h.getMessage(topic, 8) + delay := (len(messages) > 1) + + // publish all the messages + for _, msg := range messages { + // serialize here + srv(s, msg) + + // sending a backlog of messages + if delay { + time.Sleep(time.Millisecond * 100) + } + } + }() + + return nil +} + +func (h *httpBroker) Subscribe(topic string, handler Handler, opts ...SubscribeOption) (Subscriber, error) { + var err error + var host, port string + options := NewSubscribeOptions(opts...) + + // parse address for host, port + host, port, err = net.SplitHostPort(h.Address()) + if err != nil { + return nil, err + } + + addr, err := maddr.Extract(host) + if err != nil { + return nil, err + } + + var secure bool + + if h.opts.Secure || h.opts.TLSConfig != nil { + secure = true + } + + // register service + node := ®istry.Node{ + Id: topic + "-" + h.id, + Address: mnet.HostPort(addr, port), + Metadata: map[string]string{ + "secure": fmt.Sprintf("%t", secure), + "broker": "http", + "topic": topic, + }, + } + + // check for queue group or broadcast queue + version := options.Queue + if len(version) == 0 { + version = broadcastVersion + } + + service := ®istry.Service{ + Name: serviceName, + Version: version, + Nodes: []*registry.Node{node}, + } + + // generate subscriber + subscriber := &httpSubscriber{ + opts: options, + hb: h, + id: node.Id, + topic: topic, + fn: handler, + svc: service, + } + + // subscribe now + if err := h.subscribe(subscriber); err != nil { + return nil, err + } + + // return the subscriber + return subscriber, nil +} + +func (h *httpBroker) String() string { + return "http" +} + +// NewBroker returns a new http broker +func NewBroker(opts ...Option) Broker { + return newHttpBroker(opts...) +} diff --git a/broker/http/http.go b/broker/http/http.go new file mode 100644 index 00000000..ff28a41d --- /dev/null +++ b/broker/http/http.go @@ -0,0 +1,11 @@ +// Package http provides a http based message broker +package http + +import ( + "github.com/micro/go-micro/v2/broker" +) + +// NewBroker returns a new http broker +func NewBroker(opts ...broker.Option) broker.Broker { + return broker.NewBroker(opts...) +} diff --git a/broker/http/options.go b/broker/http/options.go new file mode 100644 index 00000000..c9825e1d --- /dev/null +++ b/broker/http/options.go @@ -0,0 +1,23 @@ +package http + +import ( + "context" + "net/http" + + "github.com/micro/go-micro/v2/broker" +) + +// Handle registers the handler for the given pattern. +func Handle(pattern string, handler http.Handler) broker.Option { + return func(o *broker.Options) { + if o.Context == nil { + o.Context = context.Background() + } + handlers, ok := o.Context.Value("http_handlers").(map[string]http.Handler) + if !ok { + handlers = make(map[string]http.Handler) + } + handlers[pattern] = handler + o.Context = context.WithValue(o.Context, "http_handlers", handlers) + } +} diff --git a/broker/http_test.go b/broker/http_test.go new file mode 100644 index 00000000..7d5309ae --- /dev/null +++ b/broker/http_test.go @@ -0,0 +1,384 @@ +package broker_test + +import ( + "sync" + "testing" + "time" + + "github.com/google/uuid" + "github.com/micro/go-micro/v2/broker" + "github.com/micro/go-micro/v2/registry" + "github.com/micro/go-micro/v2/registry/memory" +) + +var ( + // mock data + testData = map[string][]*registry.Service{ + "foo": { + { + Name: "foo", + Version: "1.0.0", + Nodes: []*registry.Node{ + { + Id: "foo-1.0.0-123", + Address: "localhost:9999", + }, + { + Id: "foo-1.0.0-321", + Address: "localhost:9999", + }, + }, + }, + { + Name: "foo", + Version: "1.0.1", + Nodes: []*registry.Node{ + { + Id: "foo-1.0.1-321", + Address: "localhost:6666", + }, + }, + }, + { + Name: "foo", + Version: "1.0.3", + Nodes: []*registry.Node{ + { + Id: "foo-1.0.3-345", + Address: "localhost:8888", + }, + }, + }, + }, + } +) + +func newTestRegistry() registry.Registry { + return memory.NewRegistry(memory.Services(testData)) +} + +func sub(be *testing.B, c int) { + be.StopTimer() + m := newTestRegistry() + + b := broker.NewBroker(broker.Registry(m)) + topic := uuid.New().String() + + if err := b.Init(); err != nil { + be.Fatalf("Unexpected init error: %v", err) + } + + if err := b.Connect(); err != nil { + be.Fatalf("Unexpected connect error: %v", err) + } + + msg := &broker.Message{ + Header: map[string]string{ + "Content-Type": "application/json", + }, + Body: []byte(`{"message": "Hello World"}`), + } + + var subs []broker.Subscriber + done := make(chan bool, c) + + for i := 0; i < c; i++ { + sub, err := b.Subscribe(topic, func(p broker.Event) error { + done <- true + m := p.Message() + + if string(m.Body) != string(msg.Body) { + be.Fatalf("Unexpected msg %s, expected %s", string(m.Body), string(msg.Body)) + } + + return nil + }, broker.Queue("shared")) + if err != nil { + be.Fatalf("Unexpected subscribe error: %v", err) + } + subs = append(subs, sub) + } + + for i := 0; i < be.N; i++ { + be.StartTimer() + if err := b.Publish(topic, msg); err != nil { + be.Fatalf("Unexpected publish error: %v", err) + } + <-done + be.StopTimer() + } + + for _, sub := range subs { + sub.Unsubscribe() + } + + if err := b.Disconnect(); err != nil { + be.Fatalf("Unexpected disconnect error: %v", err) + } +} + +func pub(be *testing.B, c int) { + be.StopTimer() + m := newTestRegistry() + b := broker.NewBroker(broker.Registry(m)) + topic := uuid.New().String() + + if err := b.Init(); err != nil { + be.Fatalf("Unexpected init error: %v", err) + } + + if err := b.Connect(); err != nil { + be.Fatalf("Unexpected connect error: %v", err) + } + + msg := &broker.Message{ + Header: map[string]string{ + "Content-Type": "application/json", + }, + Body: []byte(`{"message": "Hello World"}`), + } + + done := make(chan bool, c*4) + + sub, err := b.Subscribe(topic, func(p broker.Event) error { + done <- true + m := p.Message() + if string(m.Body) != string(msg.Body) { + be.Fatalf("Unexpected msg %s, expected %s", string(m.Body), string(msg.Body)) + } + return nil + }, broker.Queue("shared")) + if err != nil { + be.Fatalf("Unexpected subscribe error: %v", err) + } + + var wg sync.WaitGroup + ch := make(chan int, c*4) + be.StartTimer() + + for i := 0; i < c; i++ { + go func() { + for range ch { + if err := b.Publish(topic, msg); err != nil { + be.Fatalf("Unexpected publish error: %v", err) + } + select { + case <-done: + case <-time.After(time.Second): + } + wg.Done() + } + }() + } + + for i := 0; i < be.N; i++ { + wg.Add(1) + ch <- i + } + + wg.Wait() + be.StopTimer() + sub.Unsubscribe() + close(ch) + close(done) + + if err := b.Disconnect(); err != nil { + be.Fatalf("Unexpected disconnect error: %v", err) + } +} + +func TestBroker(t *testing.T) { + m := newTestRegistry() + b := broker.NewBroker(broker.Registry(m)) + + if err := b.Init(); err != nil { + t.Fatalf("Unexpected init error: %v", err) + } + + if err := b.Connect(); err != nil { + t.Fatalf("Unexpected connect error: %v", err) + } + + msg := &broker.Message{ + Header: map[string]string{ + "Content-Type": "application/json", + }, + Body: []byte(`{"message": "Hello World"}`), + } + + done := make(chan bool) + + sub, err := b.Subscribe("test", func(p broker.Event) error { + m := p.Message() + + if string(m.Body) != string(msg.Body) { + t.Fatalf("Unexpected msg %s, expected %s", string(m.Body), string(msg.Body)) + } + + close(done) + return nil + }) + if err != nil { + t.Fatalf("Unexpected subscribe error: %v", err) + } + + if err := b.Publish("test", msg); err != nil { + t.Fatalf("Unexpected publish error: %v", err) + } + + <-done + sub.Unsubscribe() + + if err := b.Disconnect(); err != nil { + t.Fatalf("Unexpected disconnect error: %v", err) + } +} + +func TestConcurrentSubBroker(t *testing.T) { + m := newTestRegistry() + b := broker.NewBroker(broker.Registry(m)) + + if err := b.Init(); err != nil { + t.Fatalf("Unexpected init error: %v", err) + } + + if err := b.Connect(); err != nil { + t.Fatalf("Unexpected connect error: %v", err) + } + + msg := &broker.Message{ + Header: map[string]string{ + "Content-Type": "application/json", + }, + Body: []byte(`{"message": "Hello World"}`), + } + + var subs []broker.Subscriber + var wg sync.WaitGroup + + for i := 0; i < 10; i++ { + sub, err := b.Subscribe("test", func(p broker.Event) error { + defer wg.Done() + + m := p.Message() + + if string(m.Body) != string(msg.Body) { + t.Fatalf("Unexpected msg %s, expected %s", string(m.Body), string(msg.Body)) + } + + return nil + }) + if err != nil { + t.Fatalf("Unexpected subscribe error: %v", err) + } + + wg.Add(1) + subs = append(subs, sub) + } + + if err := b.Publish("test", msg); err != nil { + t.Fatalf("Unexpected publish error: %v", err) + } + + wg.Wait() + + for _, sub := range subs { + sub.Unsubscribe() + } + + if err := b.Disconnect(); err != nil { + t.Fatalf("Unexpected disconnect error: %v", err) + } +} + +func TestConcurrentPubBroker(t *testing.T) { + m := newTestRegistry() + b := broker.NewBroker(broker.Registry(m)) + + if err := b.Init(); err != nil { + t.Fatalf("Unexpected init error: %v", err) + } + + if err := b.Connect(); err != nil { + t.Fatalf("Unexpected connect error: %v", err) + } + + msg := &broker.Message{ + Header: map[string]string{ + "Content-Type": "application/json", + }, + Body: []byte(`{"message": "Hello World"}`), + } + + var wg sync.WaitGroup + + sub, err := b.Subscribe("test", func(p broker.Event) error { + defer wg.Done() + + m := p.Message() + + if string(m.Body) != string(msg.Body) { + t.Fatalf("Unexpected msg %s, expected %s", string(m.Body), string(msg.Body)) + } + + return nil + }) + if err != nil { + t.Fatalf("Unexpected subscribe error: %v", err) + } + + for i := 0; i < 10; i++ { + wg.Add(1) + + if err := b.Publish("test", msg); err != nil { + t.Fatalf("Unexpected publish error: %v", err) + } + } + + wg.Wait() + + sub.Unsubscribe() + + if err := b.Disconnect(); err != nil { + t.Fatalf("Unexpected disconnect error: %v", err) + } +} + +func BenchmarkSub1(b *testing.B) { + sub(b, 1) +} +func BenchmarkSub8(b *testing.B) { + sub(b, 8) +} + +func BenchmarkSub32(b *testing.B) { + sub(b, 32) +} + +func BenchmarkSub64(b *testing.B) { + sub(b, 64) +} + +func BenchmarkSub128(b *testing.B) { + sub(b, 128) +} + +func BenchmarkPub1(b *testing.B) { + pub(b, 1) +} + +func BenchmarkPub8(b *testing.B) { + pub(b, 8) +} + +func BenchmarkPub32(b *testing.B) { + pub(b, 32) +} + +func BenchmarkPub64(b *testing.B) { + pub(b, 64) +} + +func BenchmarkPub128(b *testing.B) { + pub(b, 128) +} diff --git a/config/cmd/cmd.go b/config/cmd/cmd.go index 44071b71..b5cd79f8 100644 --- a/config/cmd/cmd.go +++ b/config/cmd/cmd.go @@ -36,6 +36,7 @@ import ( smucp "github.com/micro/go-micro/v2/server/mucp" // brokers + brokerHttp "github.com/micro/go-micro/v2/broker/http" "github.com/micro/go-micro/v2/broker/memory" "github.com/micro/go-micro/v2/broker/nats" brokerSrv "github.com/micro/go-micro/v2/broker/service" @@ -319,6 +320,7 @@ var ( "service": brokerSrv.NewBroker, "memory": memory.NewBroker, "nats": nats.NewBroker, + "http": brokerHttp.NewBroker, } DefaultClients = map[string]func(...client.Option) client.Client{ diff --git a/go.mod b/go.mod index 4ce7a293..9725ae12 100644 --- a/go.mod +++ b/go.mod @@ -50,8 +50,6 @@ require ( github.com/micro/mdns v0.3.0 github.com/miekg/dns v1.1.27 github.com/mitchellh/hashstructure v1.0.0 - github.com/nats-io/nats-server/v2 v2.1.4 - github.com/nats-io/nats.go v1.9.1 github.com/nlopes/slack v0.6.1-0.20191106133607-d06c2a2b3249 github.com/oxtoacart/bpool v0.0.0-20190530202638-03653db5a59c github.com/patrickmn/go-cache v2.1.0+incompatible diff --git a/go.sum b/go.sum index 63e5e123..36dee0a2 100644 --- a/go.sum +++ b/go.sum @@ -310,18 +310,6 @@ github.com/morikuni/aec v0.0.0-20170113033406-39771216ff4c h1:nXxl5PrvVm2L/wCy8d github.com/morikuni/aec v0.0.0-20170113033406-39771216ff4c/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/namedotcom/go v0.0.0-20180403034216-08470befbe04/go.mod h1:5sN+Lt1CaY4wsPvgQH/jsuJi4XO2ssZbdsIizr4CVC8= -github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg= -github.com/nats-io/jwt v0.3.2 h1:+RB5hMpXUUA2dfxuhBTEkMOrYmM+gKIZYS1KjSostMI= -github.com/nats-io/jwt v0.3.2/go.mod h1:/euKqTS1ZD+zzjYrY7pseZrTtWQSjujC7xjPc8wL6eU= -github.com/nats-io/nats-server/v2 v2.1.4 h1:BILRnsJ2Yb/fefiFbBWADpViGF69uh4sxe8poVDQ06g= -github.com/nats-io/nats-server/v2 v2.1.4/go.mod h1:Jw1Z28soD/QasIA2uWjXyM9El1jly3YwyFOuR8tH1rg= -github.com/nats-io/nats.go v1.9.1 h1:ik3HbLhZ0YABLto7iX80pZLPw/6dx3T+++MZJwLnMrQ= -github.com/nats-io/nats.go v1.9.1/go.mod h1:ZjDU1L/7fJ09jvUSRVBR2e7+RnLiiIQyqyzEE/Zbp4w= -github.com/nats-io/nkeys v0.1.0/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= -github.com/nats-io/nkeys v0.1.3 h1:6JrEfig+HzTH85yxzhSVbjHRJv9cn0p6n3IngIcM5/k= -github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= -github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= -github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32/go.mod h1:9wM+0iRr9ahx58uYLpLIr5fm8diHn0JbqRycJi6w0Ms= github.com/nlopes/slack v0.6.1-0.20191106133607-d06c2a2b3249 h1:Pr5gZa2VcmktVwq0lyC39MsN5tz356vC/pQHKvq+QBo= github.com/nlopes/slack v0.6.1-0.20191106133607-d06c2a2b3249/go.mod h1:JzQ9m3PMAqcpeCam7UaHSuBuupz7CmpjehYMayT6YOk= From 3f3d2f502750e90e7efd260b0639df7f561905cf Mon Sep 17 00:00:00 2001 From: Asim Aslam Date: Sat, 11 Apr 2020 01:51:26 +0100 Subject: [PATCH 17/44] fixup broker http address --- broker/http.go | 10 ++++++---- go.mod | 6 ++++-- go.sum | 16 ++++++++++++++++ 3 files changed, 26 insertions(+), 6 deletions(-) diff --git a/broker/http.go b/broker/http.go index a4a90698..ef06a182 100644 --- a/broker/http.go +++ b/broker/http.go @@ -65,7 +65,8 @@ type httpEvent struct { } var ( - DefaultSubPath = "/" + DefaultPath = "/" + DefaultAddress = "127.0.0.1:0" serviceName = "micro.http.broker" broadcastVersion = "ff.http.broadcast" registerTTL = time.Minute @@ -118,7 +119,8 @@ func newHttpBroker(opts ...Option) Broker { } // set address - addr := ":0" + addr := DefaultAddress + if len(options.Addrs) > 0 && len(options.Addrs[0]) > 0 { addr = options.Addrs[0] } @@ -136,7 +138,7 @@ func newHttpBroker(opts ...Option) Broker { } // specify the message handler - h.mux.Handle(DefaultSubPath, h) + h.mux.Handle(DefaultPath, h) // get optional handlers if h.opts.Context != nil { @@ -549,7 +551,7 @@ func (h *httpBroker) Publish(topic string, msg *Message, opts ...PublishOption) vals := url.Values{} vals.Add("id", node.Id) - uri := fmt.Sprintf("%s://%s%s?%s", scheme, node.Address, DefaultSubPath, vals.Encode()) + uri := fmt.Sprintf("%s://%s%s?%s", scheme, node.Address, DefaultPath, vals.Encode()) r, err := h.c.Post(uri, "application/json", bytes.NewReader(b)) if err != nil { return err diff --git a/go.mod b/go.mod index 9725ae12..cf373f7a 100644 --- a/go.mod +++ b/go.mod @@ -28,7 +28,7 @@ require ( github.com/gobwas/pool v0.2.0 // indirect github.com/gobwas/ws v1.0.3 github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e // indirect - github.com/golang/protobuf v1.3.2 + github.com/golang/protobuf v1.3.5 github.com/google/go-cmp v0.4.0 // indirect github.com/google/uuid v1.1.1 github.com/gorilla/handlers v1.4.2 @@ -50,6 +50,8 @@ require ( github.com/micro/mdns v0.3.0 github.com/miekg/dns v1.1.27 github.com/mitchellh/hashstructure v1.0.0 + github.com/nats-io/nats-server/v2 v2.1.6 + github.com/nats-io/nats.go v1.9.2 github.com/nlopes/slack v0.6.1-0.20191106133607-d06c2a2b3249 github.com/oxtoacart/bpool v0.0.0-20190530202638-03653db5a59c github.com/patrickmn/go-cache v2.1.0+incompatible @@ -61,7 +63,7 @@ require ( github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect go.etcd.io/bbolt v1.3.4 go.uber.org/zap v1.13.0 - golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d + golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59 golang.org/x/net v0.0.0-20200222125558-5a598a2470a0 golang.org/x/time v0.0.0-20191024005414-555d28b269f0 // indirect google.golang.org/grpc v1.26.0 diff --git a/go.sum b/go.sum index 36dee0a2..848d8da2 100644 --- a/go.sum +++ b/go.sum @@ -182,6 +182,8 @@ github.com/golang/protobuf v1.3.0/go.mod h1:Qd/q+1AKNOZr9uGQzbzCmRO6sUih6GTPZv6a github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/golang/protobuf v1.3.2 h1:6nsPYzhq5kReh6QImI3k5qWzO4PEbvbIW2cwSfR/6xs= github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.5 h1:F768QJ1E9tib+q5Sc8MkdJi1RxLTbRcTf8LJV56aRls= +github.com/golang/protobuf v1.3.5/go.mod h1:6O5/vntMXwX2lRkT1hjjk0nAC1IDOTvTlVgjlRvqsdk= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0 h1:0udJVsspx3VBr5FwtLhQQtuAsVc79tTq0ocGIPAU6qo= @@ -310,6 +312,18 @@ github.com/morikuni/aec v0.0.0-20170113033406-39771216ff4c h1:nXxl5PrvVm2L/wCy8d github.com/morikuni/aec v0.0.0-20170113033406-39771216ff4c/go.mod h1:BbKIizmSmc5MMPqRYbxO4ZU0S0+P200+tUnFx7PXmsc= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/namedotcom/go v0.0.0-20180403034216-08470befbe04/go.mod h1:5sN+Lt1CaY4wsPvgQH/jsuJi4XO2ssZbdsIizr4CVC8= +github.com/nats-io/jwt v0.3.2 h1:+RB5hMpXUUA2dfxuhBTEkMOrYmM+gKIZYS1KjSostMI= +github.com/nats-io/jwt v0.3.2/go.mod h1:/euKqTS1ZD+zzjYrY7pseZrTtWQSjujC7xjPc8wL6eU= +github.com/nats-io/nats-server v1.4.1 h1:Ul1oSOGNV/L8kjr4v6l2f9Yet6WY+LevH1/7cRZ/qyA= +github.com/nats-io/nats-server/v2 v2.1.6 h1:qAaHZaS8pRRNQLFaiBA1rq5WynyEGp9DFgmMfoaiXGY= +github.com/nats-io/nats-server/v2 v2.1.6/go.mod h1:BL1NOtaBQ5/y97djERRVWNouMW7GT3gxnmbE/eC8u8A= +github.com/nats-io/nats.go v1.9.2 h1:oDeERm3NcZVrPpdR/JpGdWHMv3oJ8yY30YwxKq+DU2s= +github.com/nats-io/nats.go v1.9.2/go.mod h1:AjGArbfyR50+afOUotNX2Xs5SYHf+CoOa5HH1eEl2HE= +github.com/nats-io/nkeys v0.1.3/go.mod h1:xpnFELMwJABBLVhffcfd1MZx6VsNRFpEugbxziKVo7w= +github.com/nats-io/nkeys v0.1.4 h1:aEsHIssIk6ETN5m2/MD8Y4B2X7FfXrBAUdkyRvbVYzA= +github.com/nats-io/nkeys v0.1.4/go.mod h1:XdZpAbhgyyODYqjTawOnIOI7VlbKSarI9Gfy1tqEu/s= +github.com/nats-io/nuid v1.0.1 h1:5iA8DT8V7q8WK2EScv2padNa/rTESc1KdnPw4TC2paw= +github.com/nats-io/nuid v1.0.1/go.mod h1:19wcPz3Ph3q0Jbyiqsd0kePYG7A95tJPxeL+1OSON2c= github.com/nbio/st v0.0.0-20140626010706-e9e8d9816f32/go.mod h1:9wM+0iRr9ahx58uYLpLIr5fm8diHn0JbqRycJi6w0Ms= github.com/nlopes/slack v0.6.1-0.20191106133607-d06c2a2b3249 h1:Pr5gZa2VcmktVwq0lyC39MsN5tz356vC/pQHKvq+QBo= github.com/nlopes/slack v0.6.1-0.20191106133607-d06c2a2b3249/go.mod h1:JzQ9m3PMAqcpeCam7UaHSuBuupz7CmpjehYMayT6YOk= @@ -453,6 +467,8 @@ golang.org/x/crypto v0.0.0-20190927123631-a832865fa7ad/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d h1:1ZiEyfaQIg3Qh0EoqpwAakHVhecoE5wlSg5GjnafJGw= golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59 h1:3zb4D3T4G8jdExgVU/95+vQXfpEPiMdCaZgmGVxjNHM= +golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= From c612d86480ad8f42dcc6a65b9945b9828c397b54 Mon Sep 17 00:00:00 2001 From: Asim Aslam Date: Sat, 11 Apr 2020 09:33:10 +0100 Subject: [PATCH 18/44] Move sync store --- {sync/store => store/sync}/manager.go | 12 +-- {sync/store => store/sync}/options.go | 12 +-- store/sync/sync.go | 114 ++++++++++++++++++++++++++ sync/store/cache.go | 114 -------------------------- 4 files changed, 126 insertions(+), 126 deletions(-) rename {sync/store => store/sync}/manager.go (88%) rename {sync/store => store/sync}/options.go (78%) create mode 100644 store/sync/sync.go delete mode 100644 sync/store/cache.go diff --git a/sync/store/manager.go b/store/sync/manager.go similarity index 88% rename from sync/store/manager.go rename to store/sync/manager.go index 58544aa4..6f7c5d6e 100644 --- a/sync/store/manager.go +++ b/store/sync/manager.go @@ -1,4 +1,4 @@ -package store +package sync import ( "time" @@ -25,7 +25,7 @@ const ( listOp ) -func (c *cache) cacheManager() { +func (c *syncStore) syncManager() { tickerAggregator := make(chan struct{ index int }) for i, ticker := range c.pendingWriteTickers { go func(index int, c chan struct{ index int }, t *time.Ticker) { @@ -43,18 +43,18 @@ func (c *cache) cacheManager() { } } -func (c *cache) processQueue(index int) { +func (c *syncStore) processQueue(index int) { c.Lock() defer c.Unlock() q := c.pendingWrites[index] for i := 0; i < q.Len(); i++ { r, ok := q.PopFront() if !ok { - panic(errors.Errorf("retrieved an invalid value from the L%d cache queue", index+1)) + panic(errors.Errorf("retrieved an invalid value from the L%d sync queue", index+1)) } ir, ok := r.(*internalRecord) if !ok { - panic(errors.Errorf("retrieved a non-internal record from the L%d cache queue", index+1)) + panic(errors.Errorf("retrieved a non-internal record from the L%d sync queue", index+1)) } if !ir.expiresAt.IsZero() && time.Now().After(ir.expiresAt) { continue @@ -68,7 +68,7 @@ func (c *cache) processQueue(index int) { nr.Expiry = time.Until(ir.expiresAt) } // Todo = internal queue also has to hold the corresponding store.WriteOptions - if err := c.cOptions.Stores[index+1].Write(nr); err != nil { + if err := c.syncOpts.Stores[index+1].Write(nr); err != nil { // some error, so queue for retry and bail q.PushBack(ir) return diff --git a/sync/store/options.go b/store/sync/options.go similarity index 78% rename from sync/store/options.go rename to store/sync/options.go index f2a75534..30e493f2 100644 --- a/sync/store/options.go +++ b/store/sync/options.go @@ -1,4 +1,4 @@ -package store +package sync import ( "time" @@ -6,9 +6,9 @@ import ( "github.com/micro/go-micro/v2/store" ) -// Options represents Cache options +// Options represents Sync options type Options struct { - // Stores represents layers in the cache in ascending order. L0, L1, L2, etc + // Stores represents layers in the sync in ascending order. L0, L1, L2, etc Stores []store.Store // SyncInterval is the duration between syncs from L0 to L1 SyncInterval time.Duration @@ -16,10 +16,10 @@ type Options struct { SyncMultiplier int64 } -// Option sets Cache Options +// Option sets Sync Options type Option func(o *Options) -// Stores sets the layers that make up the cache +// Stores sets the layers that make up the sync func Stores(stores ...store.Store) Option { return func(o *Options) { o.Stores = make([]store.Store, len(stores)) @@ -36,7 +36,7 @@ func SyncInterval(d time.Duration) Option { } } -// SyncMultiplier sets the multiplication factor for time to wait each cache layer +// SyncMultiplier sets the multiplication factor for time to wait each sync layer func SyncMultiplier(i int64) Option { return func(o *Options) { o.SyncMultiplier = i diff --git a/store/sync/sync.go b/store/sync/sync.go new file mode 100644 index 00000000..79a7e36e --- /dev/null +++ b/store/sync/sync.go @@ -0,0 +1,114 @@ +// Package syncs will sync multiple stores +package sync + +import ( + "fmt" + "sync" + "time" + + "github.com/ef-ds/deque" + "github.com/micro/go-micro/v2/store" + "github.com/pkg/errors" +) + +// Sync implements a sync in for stores +type Sync interface { + // Implements the store interface + store.Store + // Force a full sync + Sync() error +} +type syncStore struct { + storeOpts store.Options + syncOpts Options + pendingWrites []*deque.Deque + pendingWriteTickers []*time.Ticker + sync.RWMutex +} + +// NewSync returns a new Sync +func NewSync(opts ...Option) Sync { + c := &syncStore{} + for _, o := range opts { + o(&c.syncOpts) + } + if c.syncOpts.SyncInterval == 0 { + c.syncOpts.SyncInterval = 1 * time.Minute + } + if c.syncOpts.SyncMultiplier == 0 { + c.syncOpts.SyncMultiplier = 5 + } + return c +} + +func (c *syncStore) Close() error { + return nil +} + +// Init initialises the storeOptions +func (c *syncStore) Init(opts ...store.Option) error { + for _, o := range opts { + o(&c.storeOpts) + } + if len(c.syncOpts.Stores) == 0 { + return errors.New("the sync has no stores") + } + if c.storeOpts.Context == nil { + return errors.New("please provide a context to the sync. Cancelling the context signals that the sync is being disposed and syncs the sync") + } + for _, s := range c.syncOpts.Stores { + if err := s.Init(); err != nil { + return errors.Wrapf(err, "Store %s failed to Init()", s.String()) + } + } + c.pendingWrites = make([]*deque.Deque, len(c.syncOpts.Stores)-1) + c.pendingWriteTickers = make([]*time.Ticker, len(c.syncOpts.Stores)-1) + for i := 0; i < len(c.pendingWrites); i++ { + c.pendingWrites[i] = deque.New() + c.pendingWrites[i].Init() + c.pendingWriteTickers[i] = time.NewTicker(c.syncOpts.SyncInterval * time.Duration(intpow(c.syncOpts.SyncMultiplier, int64(i)))) + } + go c.syncManager() + return nil +} + +// Options returns the sync's store options +func (c *syncStore) Options() store.Options { + return c.storeOpts +} + +// String returns a printable string describing the sync +func (c *syncStore) String() string { + backends := make([]string, len(c.syncOpts.Stores)) + for i, s := range c.syncOpts.Stores { + backends[i] = s.String() + } + return fmt.Sprintf("sync %v", backends) +} + +func (c *syncStore) List(opts ...store.ListOption) ([]string, error) { + return c.syncOpts.Stores[0].List(opts...) +} + +func (c *syncStore) Read(key string, opts ...store.ReadOption) ([]*store.Record, error) { + return c.syncOpts.Stores[0].Read(key, opts...) +} + +func (c *syncStore) Write(r *store.Record, opts ...store.WriteOption) error { + return c.syncOpts.Stores[0].Write(r, opts...) +} + +// Delete removes a key from the sync +func (c *syncStore) Delete(key string, opts ...store.DeleteOption) error { + return c.syncOpts.Stores[0].Delete(key, opts...) +} + +func (c *syncStore) Sync() error { + return nil +} + +type internalRecord struct { + key string + value []byte + expiresAt time.Time +} diff --git a/sync/store/cache.go b/sync/store/cache.go deleted file mode 100644 index 2385e961..00000000 --- a/sync/store/cache.go +++ /dev/null @@ -1,114 +0,0 @@ -// Package store syncs multiple go-micro stores -package store - -import ( - "fmt" - "sync" - "time" - - "github.com/ef-ds/deque" - "github.com/micro/go-micro/v2/store" - "github.com/pkg/errors" -) - -// Cache implements a cache in front of go-micro Stores -type Cache interface { - store.Store - - // Force a full sync - Sync() error -} -type cache struct { - sOptions store.Options - cOptions Options - pendingWrites []*deque.Deque - pendingWriteTickers []*time.Ticker - sync.RWMutex -} - -// NewCache returns a new Cache -func NewCache(opts ...Option) Cache { - c := &cache{} - for _, o := range opts { - o(&c.cOptions) - } - if c.cOptions.SyncInterval == 0 { - c.cOptions.SyncInterval = 1 * time.Minute - } - if c.cOptions.SyncMultiplier == 0 { - c.cOptions.SyncMultiplier = 5 - } - return c -} - -func (c *cache) Close() error { - return nil -} - -// Init initialises the storeOptions -func (c *cache) Init(opts ...store.Option) error { - for _, o := range opts { - o(&c.sOptions) - } - if len(c.cOptions.Stores) == 0 { - return errors.New("the cache has no stores") - } - if c.sOptions.Context == nil { - return errors.New("please provide a context to the cache. Cancelling the context signals that the cache is being disposed and syncs the cache") - } - for _, s := range c.cOptions.Stores { - if err := s.Init(); err != nil { - return errors.Wrapf(err, "Store %s failed to Init()", s.String()) - } - } - c.pendingWrites = make([]*deque.Deque, len(c.cOptions.Stores)-1) - c.pendingWriteTickers = make([]*time.Ticker, len(c.cOptions.Stores)-1) - for i := 0; i < len(c.pendingWrites); i++ { - c.pendingWrites[i] = deque.New() - c.pendingWrites[i].Init() - c.pendingWriteTickers[i] = time.NewTicker(c.cOptions.SyncInterval * time.Duration(intpow(c.cOptions.SyncMultiplier, int64(i)))) - } - go c.cacheManager() - return nil -} - -// Options returns the cache's store options -func (c *cache) Options() store.Options { - return c.sOptions -} - -// String returns a printable string describing the cache -func (c *cache) String() string { - backends := make([]string, len(c.cOptions.Stores)) - for i, s := range c.cOptions.Stores { - backends[i] = s.String() - } - return fmt.Sprintf("cache %v", backends) -} - -func (c *cache) List(opts ...store.ListOption) ([]string, error) { - return c.cOptions.Stores[0].List(opts...) -} - -func (c *cache) Read(key string, opts ...store.ReadOption) ([]*store.Record, error) { - return c.cOptions.Stores[0].Read(key, opts...) -} - -func (c *cache) Write(r *store.Record, opts ...store.WriteOption) error { - return c.cOptions.Stores[0].Write(r, opts...) -} - -// Delete removes a key from the cache -func (c *cache) Delete(key string, opts ...store.DeleteOption) error { - return c.cOptions.Stores[0].Delete(key, opts...) -} - -func (c *cache) Sync() error { - return nil -} - -type internalRecord struct { - key string - value []byte - expiresAt time.Time -} From 6d553cb6febeeb69bb963fe962bdb9abc9294484 Mon Sep 17 00:00:00 2001 From: Asim Aslam Date: Sat, 11 Apr 2020 09:34:04 +0100 Subject: [PATCH 19/44] add whitespace --- store/sync/sync.go | 1 + 1 file changed, 1 insertion(+) diff --git a/store/sync/sync.go b/store/sync/sync.go index 79a7e36e..d1f0e240 100644 --- a/store/sync/sync.go +++ b/store/sync/sync.go @@ -18,6 +18,7 @@ type Sync interface { // Force a full sync Sync() error } + type syncStore struct { storeOpts store.Options syncOpts Options From 39470c1b1113a1f2313fb3907a49903881362c86 Mon Sep 17 00:00:00 2001 From: Asim Aslam Date: Sat, 11 Apr 2020 10:37:54 +0100 Subject: [PATCH 20/44] Completely replace sync implementation --- sync/cron.go | 99 -------------- sync/etcd/etcd.go | 179 +++++++++++++++++++++++++ sync/leader/etcd/etcd.go | 136 ------------------- sync/leader/leader.go | 25 ---- sync/leader/options.go | 22 ---- sync/lock/etcd/etcd.go | 113 ---------------- sync/lock/http/http.go | 135 ------------------- sync/lock/http/server/server.go | 45 ------- sync/lock/lock.go | 32 ----- sync/lock/memory/memory.go | 142 -------------------- sync/lock/options.go | 33 ----- sync/map.go | 156 ---------------------- sync/memory/memory.go | 202 ++++++++++++++++++++++++++++ sync/options.go | 33 +++-- sync/sync.go | 66 ++++++---- sync/task/broker/broker.go | 227 -------------------------------- sync/task/local/local.go | 59 --------- sync/task/task.go | 85 ------------ sync/time/local/local.go | 18 --- sync/time/ntp/ntp.go | 51 ------- sync/time/time.go | 18 --- 21 files changed, 435 insertions(+), 1441 deletions(-) delete mode 100644 sync/cron.go create mode 100644 sync/etcd/etcd.go delete mode 100644 sync/leader/etcd/etcd.go delete mode 100644 sync/leader/leader.go delete mode 100644 sync/leader/options.go delete mode 100644 sync/lock/etcd/etcd.go delete mode 100644 sync/lock/http/http.go delete mode 100644 sync/lock/http/server/server.go delete mode 100644 sync/lock/lock.go delete mode 100644 sync/lock/memory/memory.go delete mode 100644 sync/lock/options.go delete mode 100644 sync/map.go create mode 100644 sync/memory/memory.go delete mode 100644 sync/task/broker/broker.go delete mode 100644 sync/task/local/local.go delete mode 100644 sync/task/task.go delete mode 100644 sync/time/local/local.go delete mode 100644 sync/time/ntp/ntp.go delete mode 100644 sync/time/time.go diff --git a/sync/cron.go b/sync/cron.go deleted file mode 100644 index fddb7e49..00000000 --- a/sync/cron.go +++ /dev/null @@ -1,99 +0,0 @@ -package sync - -import ( - "fmt" - "math" - "time" - - "github.com/micro/go-micro/v2/logger" - "github.com/micro/go-micro/v2/sync/leader/etcd" - "github.com/micro/go-micro/v2/sync/task" - "github.com/micro/go-micro/v2/sync/task/local" -) - -type syncCron struct { - opts Options -} - -func backoff(attempts int) time.Duration { - if attempts == 0 { - return time.Duration(0) - } - return time.Duration(math.Pow(10, float64(attempts))) * time.Millisecond -} - -func (c *syncCron) Schedule(s task.Schedule, t task.Command) error { - id := fmt.Sprintf("%s-%s", s.String(), t.String()) - - go func() { - // run the scheduler - tc := s.Run() - - var i int - - for { - // leader election - e, err := c.opts.Leader.Elect(id) - if err != nil { - if logger.V(logger.ErrorLevel, logger.DefaultLogger) { - logger.Errorf("[cron] leader election error: %v", err) - } - time.Sleep(backoff(i)) - i++ - continue - } - - i = 0 - r := e.Revoked() - - // execute the task - Tick: - for { - select { - // schedule tick - case _, ok := <-tc: - // ticked once - if !ok { - break Tick - } - - if logger.V(logger.InfoLevel, logger.DefaultLogger) { - logger.Infof("[cron] executing command %s", t.Name) - } - if err := c.opts.Task.Run(t); err != nil { - if logger.V(logger.ErrorLevel, logger.DefaultLogger) { - logger.Errorf("[cron] error executing command %s: %v", t.Name, err) - } - } - // leader revoked - case <-r: - break Tick - } - } - - // resign - e.Resign() - } - }() - - return nil -} - -func NewCron(opts ...Option) Cron { - var options Options - for _, o := range opts { - o(&options) - } - - if options.Leader == nil { - options.Leader = etcd.NewLeader() - } - - if options.Task == nil { - options.Task = local.NewTask() - } - - return &syncCron{ - opts: options, - } -} diff --git a/sync/etcd/etcd.go b/sync/etcd/etcd.go new file mode 100644 index 00000000..7d493942 --- /dev/null +++ b/sync/etcd/etcd.go @@ -0,0 +1,179 @@ +// Package etcd is an etcd implementation of lock +package etcd + +import ( + "context" + "errors" + "log" + "path" + "strings" + gosync "sync" + + client "github.com/coreos/etcd/clientv3" + cc "github.com/coreos/etcd/clientv3/concurrency" + "github.com/micro/go-micro/v2/sync" +) + +type etcdSync struct { + options sync.Options + path string + client *client.Client + + mtx gosync.Mutex + locks map[string]*etcdLock +} + +type etcdLock struct { + s *cc.Session + m *cc.Mutex +} + +type etcdLeader struct { + opts sync.LeaderOptions + s *cc.Session + e *cc.Election + id string +} + +func (e *etcdSync) Leader(id string, opts ...sync.LeaderOption) (sync.Leader, error) { + var options sync.LeaderOptions + for _, o := range opts { + o(&options) + } + + // make path + path := path.Join(e.path, strings.Replace(e.options.Prefix+id, "/", "-", -1)) + + s, err := cc.NewSession(e.client) + if err != nil { + return nil, err + } + + l := cc.NewElection(s, path) + + if err := l.Campaign(context.TODO(), id); err != nil { + return nil, err + } + + return &etcdLeader{ + opts: options, + e: l, + id: id, + }, nil +} + +func (e *etcdLeader) Status() chan bool { + ch := make(chan bool, 1) + ech := e.e.Observe(context.Background()) + + go func() { + for r := range ech { + if string(r.Kvs[0].Value) != e.id { + ch <- true + close(ch) + return + } + } + }() + + return ch +} + +func (e *etcdLeader) Resign() error { + return e.e.Resign(context.Background()) +} + +func (e *etcdSync) Init(opts ...sync.Option) error { + for _, o := range opts { + o(&e.options) + } + return nil +} + +func (e *etcdSync) Options() sync.Options { + return e.options +} + +func (e *etcdSync) Lock(id string, opts ...sync.LockOption) error { + var options sync.LockOptions + for _, o := range opts { + o(&options) + } + + // make path + path := path.Join(e.path, strings.Replace(e.options.Prefix+id, "/", "-", -1)) + + var sopts []cc.SessionOption + if options.TTL > 0 { + sopts = append(sopts, cc.WithTTL(int(options.TTL.Seconds()))) + } + + s, err := cc.NewSession(e.client, sopts...) + if err != nil { + return err + } + + m := cc.NewMutex(s, path) + + if err := m.Lock(context.TODO()); err != nil { + return err + } + + e.mtx.Lock() + e.locks[id] = &etcdLock{ + s: s, + m: m, + } + e.mtx.Unlock() + return nil +} + +func (e *etcdSync) Unlock(id string) error { + e.mtx.Lock() + defer e.mtx.Unlock() + v, ok := e.locks[id] + if !ok { + return errors.New("lock not found") + } + err := v.m.Unlock(context.Background()) + delete(e.locks, id) + return err +} + +func (e *etcdSync) String() string { + return "etcd" +} + +func NewSync(opts ...sync.Option) sync.Sync { + var options sync.Options + for _, o := range opts { + o(&options) + } + + var endpoints []string + + for _, addr := range options.Nodes { + if len(addr) > 0 { + endpoints = append(endpoints, addr) + } + } + + if len(endpoints) == 0 { + endpoints = []string{"http://127.0.0.1:2379"} + } + + // TODO: parse addresses + c, err := client.New(client.Config{ + Endpoints: endpoints, + }) + if err != nil { + log.Fatal(err) + } + + return &etcdSync{ + path: "/micro/sync", + client: c, + options: options, + locks: make(map[string]*etcdLock), + } +} diff --git a/sync/leader/etcd/etcd.go b/sync/leader/etcd/etcd.go deleted file mode 100644 index 8e23370b..00000000 --- a/sync/leader/etcd/etcd.go +++ /dev/null @@ -1,136 +0,0 @@ -package etcd - -import ( - "context" - "log" - "path" - "strings" - - client "github.com/coreos/etcd/clientv3" - cc "github.com/coreos/etcd/clientv3/concurrency" - "github.com/micro/go-micro/v2/sync/leader" -) - -type etcdLeader struct { - opts leader.Options - path string - client *client.Client -} - -type etcdElected struct { - s *cc.Session - e *cc.Election - id string -} - -func (e *etcdLeader) Elect(id string, opts ...leader.ElectOption) (leader.Elected, error) { - var options leader.ElectOptions - for _, o := range opts { - o(&options) - } - - // make path - path := path.Join(e.path, strings.Replace(id, "/", "-", -1)) - - s, err := cc.NewSession(e.client) - if err != nil { - return nil, err - } - - l := cc.NewElection(s, path) - - if err := l.Campaign(context.TODO(), id); err != nil { - return nil, err - } - - return &etcdElected{ - e: l, - id: id, - }, nil -} - -func (e *etcdLeader) Follow() chan string { - ch := make(chan string) - - s, err := cc.NewSession(e.client) - if err != nil { - return ch - } - - l := cc.NewElection(s, e.path) - ech := l.Observe(context.Background()) - - go func() { - for r := range ech { - ch <- string(r.Kvs[0].Value) - } - }() - - return ch -} - -func (e *etcdLeader) String() string { - return "etcd" -} - -func (e *etcdElected) Reelect() error { - return e.e.Campaign(context.TODO(), e.id) -} - -func (e *etcdElected) Revoked() chan bool { - ch := make(chan bool, 1) - ech := e.e.Observe(context.Background()) - - go func() { - for r := range ech { - if string(r.Kvs[0].Value) != e.id { - ch <- true - close(ch) - return - } - } - }() - - return ch -} - -func (e *etcdElected) Resign() error { - return e.e.Resign(context.Background()) -} - -func (e *etcdElected) Id() string { - return e.id -} - -func NewLeader(opts ...leader.Option) leader.Leader { - var options leader.Options - for _, o := range opts { - o(&options) - } - - var endpoints []string - - for _, addr := range options.Nodes { - if len(addr) > 0 { - endpoints = append(endpoints, addr) - } - } - - if len(endpoints) == 0 { - endpoints = []string{"http://127.0.0.1:2379"} - } - - // TODO: parse addresses - c, err := client.New(client.Config{ - Endpoints: endpoints, - }) - if err != nil { - log.Fatal(err) - } - - return &etcdLeader{ - path: "/micro/leader", - client: c, - opts: options, - } -} diff --git a/sync/leader/leader.go b/sync/leader/leader.go deleted file mode 100644 index a4ac53f3..00000000 --- a/sync/leader/leader.go +++ /dev/null @@ -1,25 +0,0 @@ -// Package leader provides leader election -package leader - -// Leader provides leadership election -type Leader interface { - // elect leader - Elect(id string, opts ...ElectOption) (Elected, error) - // follow the leader - Follow() chan string -} - -type Elected interface { - // id of leader - Id() string - // seek re-election - Reelect() error - // resign leadership - Resign() error - // observe leadership revocation - Revoked() chan bool -} - -type Option func(o *Options) - -type ElectOption func(o *ElectOptions) diff --git a/sync/leader/options.go b/sync/leader/options.go deleted file mode 100644 index 05e92e91..00000000 --- a/sync/leader/options.go +++ /dev/null @@ -1,22 +0,0 @@ -package leader - -type Options struct { - Nodes []string - Group string -} - -type ElectOptions struct{} - -// Nodes sets the addresses of the underlying systems -func Nodes(a ...string) Option { - return func(o *Options) { - o.Nodes = a - } -} - -// Group sets the group name for coordinating leadership -func Group(g string) Option { - return func(o *Options) { - o.Group = g - } -} diff --git a/sync/lock/etcd/etcd.go b/sync/lock/etcd/etcd.go deleted file mode 100644 index 1d2a7e70..00000000 --- a/sync/lock/etcd/etcd.go +++ /dev/null @@ -1,113 +0,0 @@ -// Package etcd is an etcd implementation of lock -package etcd - -import ( - "context" - "errors" - "log" - "path" - "strings" - "sync" - - client "github.com/coreos/etcd/clientv3" - cc "github.com/coreos/etcd/clientv3/concurrency" - "github.com/micro/go-micro/v2/sync/lock" -) - -type etcdLock struct { - opts lock.Options - path string - client *client.Client - - sync.Mutex - locks map[string]*elock -} - -type elock struct { - s *cc.Session - m *cc.Mutex -} - -func (e *etcdLock) Acquire(id string, opts ...lock.AcquireOption) error { - var options lock.AcquireOptions - for _, o := range opts { - o(&options) - } - - // make path - path := path.Join(e.path, strings.Replace(e.opts.Prefix+id, "/", "-", -1)) - - var sopts []cc.SessionOption - if options.TTL > 0 { - sopts = append(sopts, cc.WithTTL(int(options.TTL.Seconds()))) - } - - s, err := cc.NewSession(e.client, sopts...) - if err != nil { - return err - } - - m := cc.NewMutex(s, path) - - if err := m.Lock(context.TODO()); err != nil { - return err - } - - e.Lock() - e.locks[id] = &elock{ - s: s, - m: m, - } - e.Unlock() - return nil -} - -func (e *etcdLock) Release(id string) error { - e.Lock() - defer e.Unlock() - v, ok := e.locks[id] - if !ok { - return errors.New("lock not found") - } - err := v.m.Unlock(context.Background()) - delete(e.locks, id) - return err -} - -func (e *etcdLock) String() string { - return "etcd" -} - -func NewLock(opts ...lock.Option) lock.Lock { - var options lock.Options - for _, o := range opts { - o(&options) - } - - var endpoints []string - - for _, addr := range options.Nodes { - if len(addr) > 0 { - endpoints = append(endpoints, addr) - } - } - - if len(endpoints) == 0 { - endpoints = []string{"http://127.0.0.1:2379"} - } - - // TODO: parse addresses - c, err := client.New(client.Config{ - Endpoints: endpoints, - }) - if err != nil { - log.Fatal(err) - } - - return &etcdLock{ - path: "/micro/lock", - client: c, - opts: options, - locks: make(map[string]*elock), - } -} diff --git a/sync/lock/http/http.go b/sync/lock/http/http.go deleted file mode 100644 index 3118c189..00000000 --- a/sync/lock/http/http.go +++ /dev/null @@ -1,135 +0,0 @@ -// Package http adds a http lock implementation -package http - -import ( - "errors" - "fmt" - "hash/crc32" - "io/ioutil" - "net/http" - "net/url" - "path/filepath" - "strings" - - "github.com/micro/go-micro/v2/sync/lock" -) - -var ( - DefaultPath = "/sync/lock" - DefaultAddress = "localhost:8080" -) - -type httpLock struct { - opts lock.Options -} - -func (h *httpLock) url(do, id string) (string, error) { - sum := crc32.ChecksumIEEE([]byte(id)) - node := h.opts.Nodes[sum%uint32(len(h.opts.Nodes))] - - // parse the host:port or whatever - uri, err := url.Parse(node) - if err != nil { - return "", err - } - - if len(uri.Scheme) == 0 { - uri.Scheme = "http" - } - - // set path - // build path - path := filepath.Join(DefaultPath, do, h.opts.Prefix, id) - uri.Path = path - - // return url - return uri.String(), nil -} - -func (h *httpLock) Acquire(id string, opts ...lock.AcquireOption) error { - var options lock.AcquireOptions - for _, o := range opts { - o(&options) - } - - uri, err := h.url("acquire", id) - if err != nil { - return err - } - - ttl := fmt.Sprintf("%d", int64(options.TTL.Seconds())) - wait := fmt.Sprintf("%d", int64(options.Wait.Seconds())) - - rsp, err := http.PostForm(uri, url.Values{ - "id": {id}, - "ttl": {ttl}, - "wait": {wait}, - }) - if err != nil { - return err - } - defer rsp.Body.Close() - - b, err := ioutil.ReadAll(rsp.Body) - if err != nil { - return err - } - - // success - if rsp.StatusCode == 200 { - return nil - } - - // return error - return errors.New(string(b)) -} - -func (h *httpLock) Release(id string) error { - uri, err := h.url("release", id) - if err != nil { - return err - } - - vals := url.Values{ - "id": {id}, - } - - req, err := http.NewRequest("DELETE", uri, strings.NewReader(vals.Encode())) - if err != nil { - return err - } - - rsp, err := http.DefaultClient.Do(req) - if err != nil { - return err - } - defer rsp.Body.Close() - - b, err := ioutil.ReadAll(rsp.Body) - if err != nil { - return err - } - - // success - if rsp.StatusCode == 200 { - return nil - } - - // return error - return errors.New(string(b)) -} - -func NewLock(opts ...lock.Option) lock.Lock { - var options lock.Options - for _, o := range opts { - o(&options) - } - - if len(options.Nodes) == 0 { - options.Nodes = []string{DefaultAddress} - } - - return &httpLock{ - opts: options, - } -} diff --git a/sync/lock/http/server/server.go b/sync/lock/http/server/server.go deleted file mode 100644 index df38fd69..00000000 --- a/sync/lock/http/server/server.go +++ /dev/null @@ -1,45 +0,0 @@ -// Package server implements the sync http server -package server - -import ( - "net/http" - - "github.com/micro/go-micro/v2/sync/lock" - lkhttp "github.com/micro/go-micro/v2/sync/lock/http" -) - -func Handler(lk lock.Lock) http.Handler { - mux := http.NewServeMux() - - mux.HandleFunc(lkhttp.DefaultPath, func(w http.ResponseWriter, r *http.Request) { - r.ParseForm() - - id := r.Form.Get("id") - if len(id) == 0 { - return - } - - switch r.Method { - case "POST": - err := lk.Acquire(id) - if err != nil { - http.Error(w, err.Error(), 500) - } - case "DELETE": - err := lk.Release(id) - if err != nil { - http.Error(w, err.Error(), 500) - } - } - }) - - return mux -} - -func Server(lk lock.Lock) *http.Server { - server := &http.Server{ - Addr: lkhttp.DefaultAddress, - Handler: Handler(lk), - } - return server -} diff --git a/sync/lock/lock.go b/sync/lock/lock.go deleted file mode 100644 index c21a9125..00000000 --- a/sync/lock/lock.go +++ /dev/null @@ -1,32 +0,0 @@ -// Package lock provides distributed locking -package lock - -import ( - "errors" - "time" -) - -var ( - ErrLockTimeout = errors.New("lock timeout") -) - -// Lock is a distributed locking interface -type Lock interface { - // Acquire a lock with given id - Acquire(id string, opts ...AcquireOption) error - // Release the lock with given id - Release(id string) error -} - -type Options struct { - Nodes []string - Prefix string -} - -type AcquireOptions struct { - TTL time.Duration - Wait time.Duration -} - -type Option func(o *Options) -type AcquireOption func(o *AcquireOptions) diff --git a/sync/lock/memory/memory.go b/sync/lock/memory/memory.go deleted file mode 100644 index 99ad1532..00000000 --- a/sync/lock/memory/memory.go +++ /dev/null @@ -1,142 +0,0 @@ -// Package memory provides a sync.Mutex implementation of the lock for local use -package memory - -import ( - "sync" - "time" - - lock "github.com/micro/go-micro/v2/sync/lock" -) - -type memoryLock struct { - sync.RWMutex - locks map[string]*mlock -} - -type mlock struct { - id string - time time.Time - ttl time.Duration - release chan bool -} - -func (m *memoryLock) Acquire(id string, opts ...lock.AcquireOption) error { - // lock our access - m.Lock() - - var options lock.AcquireOptions - for _, o := range opts { - o(&options) - } - - lk, ok := m.locks[id] - if !ok { - m.locks[id] = &mlock{ - id: id, - time: time.Now(), - ttl: options.TTL, - release: make(chan bool), - } - // unlock - m.Unlock() - return nil - } - - m.Unlock() - - // set wait time - var wait <-chan time.Time - var ttl <-chan time.Time - - // decide if we should wait - if options.Wait > time.Duration(0) { - wait = time.After(options.Wait) - } - - // check the ttl of the lock - if lk.ttl > time.Duration(0) { - // time lived for the lock - live := time.Since(lk.time) - - // set a timer for the leftover ttl - if live > lk.ttl { - // release the lock if it expired - _ = m.Release(id) - } else { - ttl = time.After(live) - } - } - -lockLoop: - for { - // wait for the lock to be released - select { - case <-lk.release: - m.Lock() - - // someone locked before us - lk, ok = m.locks[id] - if ok { - m.Unlock() - continue - } - - // got chance to lock - m.locks[id] = &mlock{ - id: id, - time: time.Now(), - ttl: options.TTL, - release: make(chan bool), - } - - m.Unlock() - - break lockLoop - case <-ttl: - // ttl exceeded - _ = m.Release(id) - // TODO: check the ttl again above - ttl = nil - // try acquire - continue - case <-wait: - return lock.ErrLockTimeout - } - } - - return nil -} - -func (m *memoryLock) Release(id string) error { - m.Lock() - defer m.Unlock() - - lk, ok := m.locks[id] - // no lock exists - if !ok { - return nil - } - - // delete the lock - delete(m.locks, id) - - select { - case <-lk.release: - return nil - default: - close(lk.release) - } - - return nil -} - -func NewLock(opts ...lock.Option) lock.Lock { - var options lock.Options - for _, o := range opts { - o(&options) - } - - return &memoryLock{ - locks: make(map[string]*mlock), - } -} diff --git a/sync/lock/options.go b/sync/lock/options.go deleted file mode 100644 index 4804ff9c..00000000 --- a/sync/lock/options.go +++ /dev/null @@ -1,33 +0,0 @@ -package lock - -import ( - "time" -) - -// Nodes sets the addresses the underlying lock implementation -func Nodes(a ...string) Option { - return func(o *Options) { - o.Nodes = a - } -} - -// Prefix sets a prefix to any lock ids used -func Prefix(p string) Option { - return func(o *Options) { - o.Prefix = p - } -} - -// TTL sets the lock ttl -func TTL(t time.Duration) AcquireOption { - return func(o *AcquireOptions) { - o.TTL = t - } -} - -// Wait sets the wait time -func Wait(t time.Duration) AcquireOption { - return func(o *AcquireOptions) { - o.Wait = t - } -} diff --git a/sync/map.go b/sync/map.go deleted file mode 100644 index 2196a4e0..00000000 --- a/sync/map.go +++ /dev/null @@ -1,156 +0,0 @@ -package sync - -import ( - "bytes" - "encoding/base64" - "encoding/json" - "fmt" - "sort" - - "github.com/micro/go-micro/v2/store" -) - -type syncMap struct { - opts Options -} - -func ekey(k interface{}) string { - b, _ := json.Marshal(k) - return base64.StdEncoding.EncodeToString(b) -} - -func (m *syncMap) Read(key, val interface{}) error { - if key == nil { - return fmt.Errorf("key is nil") - } - - kstr := ekey(key) - - // lock - if err := m.opts.Lock.Acquire(kstr); err != nil { - return err - } - defer m.opts.Lock.Release(kstr) - - // get key - kval, err := m.opts.Store.Read(kstr) - if err != nil { - return err - } - - if len(kval) == 0 { - return store.ErrNotFound - } - - // decode value - return json.Unmarshal(kval[0].Value, val) -} - -func (m *syncMap) Write(key, val interface{}) error { - if key == nil { - return fmt.Errorf("key is nil") - } - - kstr := ekey(key) - - // lock - if err := m.opts.Lock.Acquire(kstr); err != nil { - return err - } - defer m.opts.Lock.Release(kstr) - - // encode value - b, err := json.Marshal(val) - if err != nil { - return err - } - - // set key - return m.opts.Store.Write(&store.Record{ - Key: kstr, - Value: b, - }) -} - -func (m *syncMap) Delete(key interface{}) error { - if key == nil { - return fmt.Errorf("key is nil") - } - - kstr := ekey(key) - - // lock - if err := m.opts.Lock.Acquire(kstr); err != nil { - return err - } - defer m.opts.Lock.Release(kstr) - return m.opts.Store.Delete(kstr) -} - -func (m *syncMap) Iterate(fn func(key, val interface{}) error) error { - keyvals, err := m.opts.Store.Read("", store.ReadPrefix()) - if err != nil { - return err - } - - sort.Slice(keyvals, func(i, j int) bool { - return keyvals[i].Key < keyvals[j].Key - }) - - for _, keyval := range keyvals { - // lock - if err := m.opts.Lock.Acquire(keyval.Key); err != nil { - return err - } - // unlock - defer m.opts.Lock.Release(keyval.Key) - - // unmarshal value - var val interface{} - - if len(keyval.Value) > 0 && keyval.Value[0] == '{' { - if err := json.Unmarshal(keyval.Value, &val); err != nil { - return err - } - } else { - val = keyval.Value - } - - // exec func - if err := fn(keyval.Key, val); err != nil { - return err - } - - // save val - b, err := json.Marshal(val) - if err != nil { - return err - } - - // no save - if i := bytes.Compare(keyval.Value, b); i == 0 { - return nil - } - - // set key - if err := m.opts.Store.Write(&store.Record{ - Key: keyval.Key, - Value: b, - }); err != nil { - return err - } - } - - return nil -} - -func NewMap(opts ...Option) Map { - var options Options - for _, o := range opts { - o(&options) - } - - return &syncMap{ - opts: options, - } -} diff --git a/sync/memory/memory.go b/sync/memory/memory.go new file mode 100644 index 00000000..f3a92180 --- /dev/null +++ b/sync/memory/memory.go @@ -0,0 +1,202 @@ +// Package memory provides a sync.Mutex implementation of the lock for local use +package memory + +import ( + gosync "sync" + "time" + + "github.com/micro/go-micro/v2/sync" +) + +type memorySync struct { + options sync.Options + + mtx gosync.RWMutex + locks map[string]*memoryLock +} + +type memoryLock struct { + id string + time time.Time + ttl time.Duration + release chan bool +} + +type memoryLeader struct { + opts sync.LeaderOptions + id string + resign func(id string) error + status chan bool +} + +func (m *memoryLeader) Resign() error { + return m.resign(m.id) +} + +func (m *memoryLeader) Status() chan bool { + return m.status +} + +func (m *memorySync) Leader(id string, opts ...sync.LeaderOption) (sync.Leader, error) { + var once gosync.Once + var options sync.LeaderOptions + for _, o := range opts { + o(&options) + } + + // acquire a lock for the id + if err := m.Lock(id); err != nil { + return nil, err + } + + // return the leader + return &memoryLeader{ + opts: options, + id: id, + resign: func(id string) error { + once.Do(func() { + m.Unlock(id) + }) + return nil + }, + // TODO: signal when Unlock is called + status: make(chan bool, 1), + }, nil +} + +func (m *memorySync) Init(opts ...sync.Option) error { + for _, o := range opts { + o(&m.options) + } + return nil +} + +func (m *memorySync) Options() sync.Options { + return m.options +} + +func (m *memorySync) Lock(id string, opts ...sync.LockOption) error { + // lock our access + m.mtx.Lock() + + var options sync.LockOptions + for _, o := range opts { + o(&options) + } + + lk, ok := m.locks[id] + if !ok { + m.locks[id] = &memoryLock{ + id: id, + time: time.Now(), + ttl: options.TTL, + release: make(chan bool), + } + // unlock + m.mtx.Unlock() + return nil + } + + m.mtx.Unlock() + + // set wait time + var wait <-chan time.Time + var ttl <-chan time.Time + + // decide if we should wait + if options.Wait > time.Duration(0) { + wait = time.After(options.Wait) + } + + // check the ttl of the lock + if lk.ttl > time.Duration(0) { + // time lived for the lock + live := time.Since(lk.time) + + // set a timer for the leftover ttl + if live > lk.ttl { + // release the lock if it expired + _ = m.Unlock(id) + } else { + ttl = time.After(live) + } + } + +lockLoop: + for { + // wait for the lock to be released + select { + case <-lk.release: + m.mtx.Lock() + + // someone locked before us + lk, ok = m.locks[id] + if ok { + m.mtx.Unlock() + continue + } + + // got chance to lock + m.locks[id] = &memoryLock{ + id: id, + time: time.Now(), + ttl: options.TTL, + release: make(chan bool), + } + + m.mtx.Unlock() + + break lockLoop + case <-ttl: + // ttl exceeded + _ = m.Unlock(id) + // TODO: check the ttl again above + ttl = nil + // try acquire + continue + case <-wait: + return sync.ErrLockTimeout + } + } + + return nil +} + +func (m *memorySync) Unlock(id string) error { + m.mtx.Lock() + defer m.mtx.Unlock() + + lk, ok := m.locks[id] + // no lock exists + if !ok { + return nil + } + + // delete the lock + delete(m.locks, id) + + select { + case <-lk.release: + return nil + default: + close(lk.release) + } + + return nil +} + +func (m *memorySync) String() string { + return "memory" +} + +func NewSync(opts ...sync.Option) sync.Sync { + var options sync.Options + for _, o := range opts { + o(&options) + } + + return &memorySync{ + options: options, + locks: make(map[string]*memoryLock), + } +} diff --git a/sync/options.go b/sync/options.go index a179479d..6c546356 100644 --- a/sync/options.go +++ b/sync/options.go @@ -1,36 +1,33 @@ package sync import ( - "github.com/micro/go-micro/v2/store" - "github.com/micro/go-micro/v2/sync/leader" - "github.com/micro/go-micro/v2/sync/lock" - "github.com/micro/go-micro/v2/sync/time" + "time" ) -// WithLeader sets the leader election implementation opton -func WithLeader(l leader.Leader) Option { +// Nodes sets the addresses to use +func Nodes(a ...string) Option { return func(o *Options) { - o.Leader = l + o.Nodes = a } } -// WithLock sets the locking implementation option -func WithLock(l lock.Lock) Option { +// Prefix sets a prefix to any lock ids used +func Prefix(p string) Option { return func(o *Options) { - o.Lock = l + o.Prefix = p } } -// WithStore sets the store implementation option -func WithStore(s store.Store) Option { - return func(o *Options) { - o.Store = s +// LockTTL sets the lock ttl +func LockTTL(t time.Duration) LockOption { + return func(o *LockOptions) { + o.TTL = t } } -// WithTime sets the time implementation option -func WithTime(t time.Time) Option { - return func(o *Options) { - o.Time = t +// LockWait sets the wait time +func LockWait(t time.Duration) LockOption { + return func(o *LockOptions) { + o.Wait = t } } diff --git a/sync/sync.go b/sync/sync.go index 69d0c410..94d62043 100644 --- a/sync/sync.go +++ b/sync/sync.go @@ -2,40 +2,52 @@ package sync import ( - "github.com/micro/go-micro/v2/store" - "github.com/micro/go-micro/v2/sync/leader" - "github.com/micro/go-micro/v2/sync/lock" - "github.com/micro/go-micro/v2/sync/task" - "github.com/micro/go-micro/v2/sync/time" + "errors" + "time" ) -// Map provides synchronized access to key-value storage. -// It uses the store interface and lock interface to -// provide a consistent storage mechanism. -type Map interface { - // Read value with given key - Read(key, val interface{}) error - // Write value with given key - Write(key, val interface{}) error - // Delete value with given key - Delete(key interface{}) error - // Iterate over all key/vals. Value changes are saved - Iterate(func(key, val interface{}) error) error +var ( + ErrLockTimeout = errors.New("lock timeout") +) + +// Sync is an interface for synchronization +type Sync interface { + // Initialise options + Init(...Option) error + // Return the options + Options() Options + // Elect a leader + Leader(id string, opts ...LeaderOption) (Leader, error) + // Lock acquires a lock + Lock(id string, opts ...LockOption) error + // Unlock releases a lock + Unlock(id string) error + // Sync implementation + String() string } -// Cron is a distributed scheduler using leader election -// and distributed task runners. It uses the leader and -// task interfaces. -type Cron interface { - Schedule(task.Schedule, task.Command) error +// Leader provides leadership election +type Leader interface { + // resign leadership + Resign() error + // status returns when leadership is lost + Status() chan bool } type Options struct { - Leader leader.Leader - Lock lock.Lock - Store store.Store - Task task.Task - Time time.Time + Nodes []string + Prefix string } type Option func(o *Options) + +type LeaderOptions struct {} + +type LeaderOption func(o *LeaderOptions) + +type LockOptions struct { + TTL time.Duration + Wait time.Duration +} + +type LockOption func(o *LockOptions) diff --git a/sync/task/broker/broker.go b/sync/task/broker/broker.go deleted file mode 100644 index feedb0ec..00000000 --- a/sync/task/broker/broker.go +++ /dev/null @@ -1,227 +0,0 @@ -// Package broker provides a distributed task manager built on the micro broker -package broker - -import ( - "context" - "errors" - "fmt" - "math/rand" - "strings" - "sync" - "time" - - "github.com/google/uuid" - "github.com/micro/go-micro/v2/broker" - "github.com/micro/go-micro/v2/sync/task" -) - -type brokerKey struct{} - -// Task is a broker task -type Task struct { - // a micro broker - Broker broker.Broker - // Options - Options task.Options - - mtx sync.RWMutex - status string -} - -func returnError(err error, ch chan error) { - select { - case ch <- err: - default: - } -} - -func (t *Task) Run(c task.Command) error { - // connect - t.Broker.Connect() - // unique id for this runner - id := uuid.New().String() - // topic of the command - topic := fmt.Sprintf("task.%s", c.Name) - - // global error - errCh := make(chan error, t.Options.Pool) - - // subscribe for distributed work - workFn := func(p broker.Event) error { - msg := p.Message() - - // get command name - command := msg.Header["Command"] - - // check the command is what we expect - if command != c.Name { - returnError(errors.New("received unknown command: "+command), errCh) - return nil - } - - // new task created - switch msg.Header["Status"] { - case "start": - // artificially delay start of processing - time.Sleep(time.Millisecond * time.Duration(10+rand.Intn(100))) - - // execute the function - err := c.Func() - - status := "done" - errors := "" - - if err != nil { - status = "error" - errors = err.Error() - } - - // create response - msg := &broker.Message{ - Header: map[string]string{ - "Command": c.Name, - "Error": errors, - "Id": id, - "Status": status, - "Timestamp": fmt.Sprintf("%d", time.Now().Unix()), - }, - // Body is nil, may be used in future - } - - // publish end of task - if err := t.Broker.Publish(topic, msg); err != nil { - returnError(err, errCh) - } - } - - return nil - } - - // subscribe for the pool size - for i := 0; i < t.Options.Pool; i++ { - err := func() error { - // subscribe to work - subWork, err := t.Broker.Subscribe(topic, workFn, broker.Queue(fmt.Sprintf("work.%d", i))) - if err != nil { - return err - } - - // unsubscribe on completion - defer subWork.Unsubscribe() - - return nil - }() - - if err != nil { - return err - } - } - - // subscribe to all status messages - subStatus, err := t.Broker.Subscribe(topic, func(p broker.Event) error { - msg := p.Message() - - // get command name - command := msg.Header["Command"] - - // check the command is what we expect - if command != c.Name { - return nil - } - - // check task status - switch msg.Header["Status"] { - // task is complete - case "done": - errCh <- nil - // someone failed - case "error": - returnError(errors.New(msg.Header["Error"]), errCh) - } - - return nil - }) - if err != nil { - return err - } - defer subStatus.Unsubscribe() - - // a new task - msg := &broker.Message{ - Header: map[string]string{ - "Command": c.Name, - "Id": id, - "Status": "start", - "Timestamp": fmt.Sprintf("%d", time.Now().Unix()), - }, - } - - // artificially delay the start of the task - time.Sleep(time.Millisecond * time.Duration(10+rand.Intn(100))) - - // publish the task - if err := t.Broker.Publish(topic, msg); err != nil { - return err - } - - var gerrors []string - - // wait for all responses - for i := 0; i < t.Options.Pool; i++ { - // check errors - err := <-errCh - - // append to errors - if err != nil { - gerrors = append(gerrors, err.Error()) - } - } - - // return the errors - if len(gerrors) > 0 { - return errors.New("errors: " + strings.Join(gerrors, "\n")) - } - - return nil -} - -func (t *Task) Status() string { - t.mtx.RLock() - defer t.mtx.RUnlock() - return t.status -} - -// Broker sets the micro broker -func WithBroker(b broker.Broker) task.Option { - return func(o *task.Options) { - if o.Context == nil { - o.Context = context.Background() - } - o.Context = context.WithValue(o.Context, brokerKey{}, b) - } -} - -// NewTask returns a new broker task -func NewTask(opts ...task.Option) task.Task { - options := task.Options{ - Context: context.Background(), - } - - for _, o := range opts { - o(&options) - } - - if options.Pool == 0 { - options.Pool = 1 - } - - b, ok := options.Context.Value(brokerKey{}).(broker.Broker) - if !ok { - b = broker.DefaultBroker - } - - return &Task{ - Broker: b, - Options: options, - } -} diff --git a/sync/task/local/local.go b/sync/task/local/local.go deleted file mode 100644 index 5ce2631d..00000000 --- a/sync/task/local/local.go +++ /dev/null @@ -1,59 +0,0 @@ -// Package local provides a local task runner -package local - -import ( - "fmt" - "sync" - - "github.com/micro/go-micro/v2/sync/task" -) - -type localTask struct { - opts task.Options - mtx sync.RWMutex - status string -} - -func (l *localTask) Run(t task.Command) error { - ch := make(chan error, l.opts.Pool) - - for i := 0; i < l.opts.Pool; i++ { - go func() { - ch <- t.Execute() - }() - } - - var err error - - for i := 0; i < l.opts.Pool; i++ { - er := <-ch - if err != nil { - err = er - l.mtx.Lock() - l.status = fmt.Sprintf("command [%s] status: %s", t.Name, err.Error()) - l.mtx.Unlock() - } - } - - close(ch) - return err -} - -func (l *localTask) Status() string { - l.mtx.RLock() - defer l.mtx.RUnlock() - return l.status -} - -func NewTask(opts ...task.Option) task.Task { - var options task.Options - for _, o := range opts { - o(&options) - } - if options.Pool == 0 { - options.Pool = 1 - } - return &localTask{ - opts: options, - } -} diff --git a/sync/task/task.go b/sync/task/task.go deleted file mode 100644 index c9b9b5d6..00000000 --- a/sync/task/task.go +++ /dev/null @@ -1,85 +0,0 @@ -// Package task provides an interface for distributed jobs -package task - -import ( - "context" - "fmt" - "time" -) - -// Task represents a distributed task -type Task interface { - // Run runs a command immediately until completion - Run(Command) error - // Status provides status of last execution - Status() string -} - -// Command to be executed -type Command struct { - Name string - Func func() error -} - -// Schedule represents a time or interval at which a task should run -type Schedule struct { - // When to start the schedule. Zero time means immediately - Time time.Time - // Non zero interval dictates an ongoing schedule - Interval time.Duration -} - -type Options struct { - // Pool size for workers - Pool int - // Alternative options - Context context.Context -} - -type Option func(o *Options) - -func (c Command) Execute() error { - return c.Func() -} - -func (c Command) String() string { - return c.Name -} - -func (s Schedule) Run() <-chan time.Time { - d := s.Time.Sub(time.Now()) - - ch := make(chan time.Time, 1) - - go func() { - // wait for start time - <-time.After(d) - - // zero interval - if s.Interval == time.Duration(0) { - ch <- time.Now() - close(ch) - return - } - - // start ticker - ticker := time.NewTicker(s.Interval) - defer ticker.Stop() - for t := range ticker.C { - ch <- t - } - }() - - return ch -} - -func (s Schedule) String() string { - return fmt.Sprintf("%d-%d", s.Time.Unix(), s.Interval) -} - -// WithPool sets the pool size for concurrent work -func WithPool(i int) Option { - return func(o *Options) { - o.Pool = i - } -} diff --git a/sync/time/local/local.go b/sync/time/local/local.go deleted file mode 100644 index 6be36de7..00000000 --- a/sync/time/local/local.go +++ /dev/null @@ -1,18 +0,0 @@ -// Package local provides a local clock -package local - -import ( - gotime "time" - - "github.com/micro/go-micro/v2/sync/time" -) - -type Time struct{} - -func (t *Time) Now() (gotime.Time, error) { - return gotime.Now(), nil -} - -func NewTime(opts ...time.Option) time.Time { - return new(Time) -} diff --git a/sync/time/ntp/ntp.go b/sync/time/ntp/ntp.go deleted file mode 100644 index 91cf5862..00000000 --- a/sync/time/ntp/ntp.go +++ /dev/null @@ -1,51 +0,0 @@ -// Package ntp provides ntp synchronized time -package ntp - -import ( - "context" - gotime "time" - - "github.com/beevik/ntp" - "github.com/micro/go-micro/v2/sync/time" -) - -type ntpTime struct { - server string -} - -type ntpServerKey struct{} - -func (n *ntpTime) Now() (gotime.Time, error) { - return ntp.Time(n.server) -} - -// NewTime returns ntp time -func NewTime(opts ...time.Option) time.Time { - options := time.Options{ - Context: context.Background(), - } - - for _, o := range opts { - o(&options) - } - - server := "time.google.com" - - if k, ok := options.Context.Value(ntpServerKey{}).(string); ok { - server = k - } - - return &ntpTime{ - server: server, - } -} - -// WithServer sets the ntp server -func WithServer(s string) time.Option { - return func(o *time.Options) { - if o.Context == nil { - o.Context = context.Background() - } - o.Context = context.WithValue(o.Context, ntpServerKey{}, s) - } -} diff --git a/sync/time/time.go b/sync/time/time.go deleted file mode 100644 index 7c11a751..00000000 --- a/sync/time/time.go +++ /dev/null @@ -1,18 +0,0 @@ -// Package time provides clock synchronization -package time - -import ( - "context" - "time" -) - -// Time returns synchronized time -type Time interface { - Now() (time.Time, error) -} - -type Options struct { - Context context.Context -} - -type Option func(o *Options) From b887d91f94078eeb65698f3368db80e1b7b5a2f2 Mon Sep 17 00:00:00 2001 From: Asim Aslam Date: Sat, 11 Apr 2020 10:38:13 +0100 Subject: [PATCH 21/44] remove readme --- sync/README.md | 116 ------------------------------------------------- 1 file changed, 116 deletions(-) delete mode 100644 sync/README.md diff --git a/sync/README.md b/sync/README.md deleted file mode 100644 index d69c2060..00000000 --- a/sync/README.md +++ /dev/null @@ -1,116 +0,0 @@ -# Sync - -Sync is a synchronization library for distributed systems. - -## Overview - -Distributed systems by their very nature are decoupled and independent. In most cases they must honour 2 out of 3 letters of the CAP theorem -e.g Availability and Partitional tolerance but sacrificing consistency. In the case of microservices we often offload this concern to -an external database or eventing system. Go Sync provides a framework for synchronization which can be used in the application by the developer. - -## Getting Started - -- [Leader](#leader) - leadership election for group coordination -- [Lock](#lock) - distributed locking for exclusive resource access -- [Task](#task) - distributed job execution -- [Time](#time) - provides synchronized time - -## Lock - -The Lock interface provides distributed locking. Multiple instances attempting to lock the same id will block until available. - -```go -import "github.com/micro/go-micro/sync/lock/consul" - -lock := consul.NewLock() - -// acquire lock -err := lock.Acquire("id") -// handle err - -// release lock -err = lock.Release("id") -// handle err -``` - -## Leader - -Leader provides leadership election. Useful where one node needs to coordinate some action. - -```go -import ( - "github.com/micro/go-micro/sync/leader" - "github.com/micro/go-micro/sync/leader/consul" -) - -l := consul.NewLeader( - leader.Group("name"), -) - -// elect leader -e, err := l.Elect("id") -// handle err - - -// operate while leader -revoked := e.Revoked() - -for { - select { - case <-revoked: - // re-elect - e.Elect("id") - default: - // leader operation - } -} - -// resign leadership -e.Resign() -``` - -## Task - -Task provides distributed job execution. It's a simple way to distribute work across a coordinated pool of workers. - -```go -import ( - "github.com/micro/go-micro/sync/task" - "github.com/micro/go-micro/sync/task/local" -) - -t := local.NewTask( - task.WithPool(10), -) - -err := t.Run(task.Command{ - Name: "atask", - Func: func() error { - // exec some work - return nil - }, -}) - -if err != nil { - // do something -} -``` - -## Time - -Time provides synchronized time. Local machines may have clock skew and time cannot be guaranteed to be the same everywhere. -Synchronized Time allows you to decide how time is defined for your applications. - -```go -import ( - "github.com/micro/go-micro/sync/time/ntp" -) - - -t := ntp.NewTime() -time, err := t.Now() -``` - -## TODO - -- Event package - strongly consistent event stream e.g kafka From c697eed1be8be7060c84008f3c145af7111dacd9 Mon Sep 17 00:00:00 2001 From: Asim Aslam Date: Sat, 11 Apr 2020 10:48:32 +0100 Subject: [PATCH 22/44] Update comments --- sync/sync.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sync/sync.go b/sync/sync.go index 94d62043..ebaec6da 100644 --- a/sync/sync.go +++ b/sync/sync.go @@ -1,4 +1,4 @@ -// Package sync is a distributed synchronization framework +// Package sync is an interface for distributed synchronization package sync import ( @@ -10,7 +10,7 @@ var ( ErrLockTimeout = errors.New("lock timeout") ) -// Sync is an interface for synchronization +// Sync is an interface for distributed synchronization type Sync interface { // Initialise options Init(...Option) error From 0f2006ac508f3ab69349349453496a5be9325200 Mon Sep 17 00:00:00 2001 From: Asim Aslam Date: Sat, 11 Apr 2020 11:02:06 +0100 Subject: [PATCH 23/44] fix compilation issues --- api/server/acme/certmagic/storage.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/api/server/acme/certmagic/storage.go b/api/server/acme/certmagic/storage.go index dc805e98..4e26b888 100644 --- a/api/server/acme/certmagic/storage.go +++ b/api/server/acme/certmagic/storage.go @@ -11,7 +11,7 @@ import ( "github.com/mholt/certmagic" "github.com/micro/go-micro/v2/store" - "github.com/micro/go-micro/v2/sync/lock" + "github.com/micro/go-micro/v2/sync" ) // File represents a "File" that will be stored in store.Store - the contents and last modified time @@ -26,16 +26,16 @@ type File struct { // As certmagic storage expects a filesystem (with stat() abilities) we have to implement // the bare minimum of metadata. type storage struct { - lock lock.Lock + lock sync.Sync store store.Store } func (s *storage) Lock(key string) error { - return s.lock.Acquire(key, lock.TTL(10*time.Minute)) + return s.lock.Lock(key, sync.LockTTL(10*time.Minute)) } func (s *storage) Unlock(key string) error { - return s.lock.Release(key) + return s.lock.Unlock(key) } func (s *storage) Store(key string, value []byte) error { @@ -139,7 +139,7 @@ func (s *storage) Stat(key string) (certmagic.KeyInfo, error) { } // NewStorage returns a certmagic.Storage backed by a go-micro/lock and go-micro/store -func NewStorage(lock lock.Lock, store store.Store) certmagic.Storage { +func NewStorage(lock sync.Sync, store store.Store) certmagic.Storage { return &storage{ lock: lock, store: store, From ac8b6f944ecabc7ec1b8c3a66cd60376698ab960 Mon Sep 17 00:00:00 2001 From: Asim Aslam Date: Sat, 11 Apr 2020 11:15:01 +0100 Subject: [PATCH 24/44] Prefix logs dir micro/logs for runtime --- runtime/default.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/runtime/default.go b/runtime/default.go index a0ea161b..e8b567c5 100644 --- a/runtime/default.go +++ b/runtime/default.go @@ -178,7 +178,7 @@ func (r *runtime) run(events <-chan Event) { func logFile(serviceName string) string { name := strings.Replace(serviceName, "/", "-", -1) - return filepath.Join(os.TempDir(), fmt.Sprintf("%v.log", name)) + return filepath.Join(os.TempDir(), "micro", "logs", fmt.Sprintf("%v.log", name)) } // Create creates a new service which is then started by runtime From c8782375675b7430a879da9fed16fcaa576628bb Mon Sep 17 00:00:00 2001 From: Asim Aslam Date: Sat, 11 Apr 2020 11:22:02 +0100 Subject: [PATCH 25/44] fix log file creation --- runtime/default.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/runtime/default.go b/runtime/default.go index e8b567c5..2ef813ff 100644 --- a/runtime/default.go +++ b/runtime/default.go @@ -40,6 +40,10 @@ func NewRuntime(opts ...Option) Runtime { o(&options) } + // make the logs directory + path := filepath.Join(os.TempDir(), "micro", "logs") + _ = os.MkdirAll(path, 0755) + return &runtime{ options: options, closed: make(chan bool), @@ -177,8 +181,10 @@ func (r *runtime) run(events <-chan Event) { } func logFile(serviceName string) string { + // make the directory name := strings.Replace(serviceName, "/", "-", -1) - return filepath.Join(os.TempDir(), "micro", "logs", fmt.Sprintf("%v.log", name)) + path := filepath.Join(os.TempDir(), "micro", "logs") + return filepath.Join(path, fmt.Sprintf("%v.log", name)) } // Create creates a new service which is then started by runtime From bb1ccf09e8b33581fe2d76091caa1923ebd53df3 Mon Sep 17 00:00:00 2001 From: Asim Aslam Date: Sat, 11 Apr 2020 11:23:41 +0100 Subject: [PATCH 26/44] prefix store dir --- store/file/file.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/store/file/file.go b/store/file/file.go index cddcaf80..5f0fc95e 100644 --- a/store/file/file.go +++ b/store/file/file.go @@ -21,7 +21,7 @@ var ( // DefaultTable when none is specified DefaultTable = "micro" // DefaultDir is the default directory for bbolt files - DefaultDir = os.TempDir() + DefaultDir = filepath.Join(os.TempDir(), "micro", "store") // bucket used for data storage dataBucket = "data" From 3f81f685dfb584005705890ff0740b932509c033 Mon Sep 17 00:00:00 2001 From: Asim Aslam Date: Sat, 11 Apr 2020 12:00:34 +0100 Subject: [PATCH 27/44] Move sync --- {store => util}/sync/manager.go | 0 {store => util}/sync/options.go | 0 {store => util}/sync/sync.go | 0 3 files changed, 0 insertions(+), 0 deletions(-) rename {store => util}/sync/manager.go (100%) rename {store => util}/sync/options.go (100%) rename {store => util}/sync/sync.go (100%) diff --git a/store/sync/manager.go b/util/sync/manager.go similarity index 100% rename from store/sync/manager.go rename to util/sync/manager.go diff --git a/store/sync/options.go b/util/sync/options.go similarity index 100% rename from store/sync/options.go rename to util/sync/options.go diff --git a/store/sync/sync.go b/util/sync/sync.go similarity index 100% rename from store/sync/sync.go rename to util/sync/sync.go From 51d4f737b8129b76c953dbef757a3479e4549a48 Mon Sep 17 00:00:00 2001 From: Asim Aslam Date: Sat, 11 Apr 2020 12:10:19 +0100 Subject: [PATCH 28/44] fixup store cache# --- store/cache/cache.go | 39 ++++++++++++++++++++++++++++----------- store/cache/cache_test.go | 7 +++---- 2 files changed, 31 insertions(+), 15 deletions(-) diff --git a/store/cache/cache.go b/store/cache/cache.go index 96790b7a..996d96e1 100644 --- a/store/cache/cache.go +++ b/store/cache/cache.go @@ -5,21 +5,34 @@ import ( "fmt" "github.com/micro/go-micro/v2/store" + "github.com/micro/go-micro/v2/store/memory" "github.com/pkg/errors" ) type cache struct { - stores []store.Store - options store.Options + stores []store.Store +} + +// Cache is a cpu register style cache for the store. +// It syncs between N stores in a faulting manner. +type Cache interface { + // Implements the store interface + store.Store } // NewCache returns a new store using the underlying stores, which must be already Init()ialised -func NewCache(stores ...store.Store) store.Store { - c := &cache{} - c.stores = make([]store.Store, len(stores)) - for i, s := range stores { - c.stores[i] = s +func NewCache(stores ...store.Store) Cache { + if len(stores) == 0 { + stores = []store.Store{ + memory.NewStore(), + } } + + // TODO: build in an in memory cache + c := &cache{ + stores: stores, + } + return c } @@ -27,15 +40,19 @@ func (c *cache) Close() error { return nil } -func (c *cache) Init(...store.Option) error { - if len(c.stores) < 2 { - return errors.New("cache requires at least 2 stores") +func (c *cache) Init(opts ...store.Option) error { + // pass to the stores + for _, store := range c.stores { + if err := store.Init(opts...); err != nil { + return err + } } return nil } func (c *cache) Options() store.Options { - return c.options + // return from first store + return c.stores[0].Options() } func (c *cache) String() string { diff --git a/store/cache/cache_test.go b/store/cache/cache_test.go index e58b2b0b..ddfa5494 100644 --- a/store/cache/cache_test.go +++ b/store/cache/cache_test.go @@ -15,13 +15,12 @@ func TestCache(t *testing.T) { assert := assert.New(t) - nonCache := NewCache(l0) - assert.NotNil(nonCache.Init(), "Expected a cache initialised with just 1 store to fail") + nonCache := NewCache(nil) + assert.Equal(len(nonCache.(*cache).stores), 1, "Expected a cache initialised with just 1 store to fail") // Basic functionality cachedStore := NewCache(l0, l1, l2) - assert.Nil(cachedStore.Init(), "Init should not error") - assert.Equal(cachedStore.Options(), store.Options{}, "Options on store/cache are nonsensical") + assert.Equal(cachedStore.Options(), l0.Options(), "Options on store/cache are nonsensical") expectedString := "cache [memory memory memory]" assert.Equal(cachedStore.String(), expectedString, "Cache couldn't describe itself as expected") From ea2bb0275c7f2a6afb1fec8bd2269adb6c818f04 Mon Sep 17 00:00:00 2001 From: Asim Aslam Date: Sat, 11 Apr 2020 13:02:53 +0100 Subject: [PATCH 29/44] Strip external use of mdns --- go.mod | 2 - go.sum | 10 - registry/mdns_registry.go | 2 +- registry/mdns_watcher.go | 2 +- util/mdns/.gitignore | 23 ++ util/mdns/client.go | 501 ++++++++++++++++++++++++++++++++++++++ util/mdns/dns_sd.go | 84 +++++++ util/mdns/dns_sd_test.go | 68 ++++++ util/mdns/server.go | 476 ++++++++++++++++++++++++++++++++++++ util/mdns/server_test.go | 61 +++++ util/mdns/zone.go | 309 +++++++++++++++++++++++ util/mdns/zone_test.go | 275 +++++++++++++++++++++ 12 files changed, 1799 insertions(+), 14 deletions(-) create mode 100644 util/mdns/.gitignore create mode 100644 util/mdns/client.go create mode 100644 util/mdns/dns_sd.go create mode 100644 util/mdns/dns_sd_test.go create mode 100644 util/mdns/server.go create mode 100644 util/mdns/server_test.go create mode 100644 util/mdns/zone.go create mode 100644 util/mdns/zone_test.go diff --git a/go.mod b/go.mod index cf373f7a..8a445a11 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,6 @@ go 1.13 require ( github.com/BurntSushi/toml v0.3.1 - github.com/beevik/ntp v0.2.0 github.com/bitly/go-simplejson v0.5.0 github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 // indirect github.com/bwmarrin/discordgo v0.20.2 @@ -47,7 +46,6 @@ require ( github.com/lucas-clemente/quic-go v0.14.1 github.com/mholt/certmagic v0.9.3 github.com/micro/cli/v2 v2.1.2 - github.com/micro/mdns v0.3.0 github.com/miekg/dns v1.1.27 github.com/mitchellh/hashstructure v1.0.0 github.com/nats-io/nats-server/v2 v2.1.6 diff --git a/go.sum b/go.sum index 848d8da2..db9453b6 100644 --- a/go.sum +++ b/go.sum @@ -53,8 +53,6 @@ github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 h1:0CwZNZbxp69SHPd github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= github.com/aws/aws-sdk-go v1.23.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/baiyubin/aliyun-sts-go-sdk v0.0.0-20180326062324-cfa1a18b161f/go.mod h1:AuiFmCCPBSrqvVMvuqFuk0qogytodnVFVSN5CeJB8Gc= -github.com/beevik/ntp v0.2.0 h1:sGsd+kAXzT0bfVfzJfce04g+dSRfrs+tbQW8lweuYgw= -github.com/beevik/ntp v0.2.0/go.mod h1:hIHWr+l3+/clUnF44zdK+CWW7fO8dR5cIylAQ76NRpg= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -290,9 +288,6 @@ github.com/mholt/certmagic v0.9.3 h1:RmzuNJ5mpFplDbyS41z+gGgE/py24IX6m0nHZ0yNTQU github.com/mholt/certmagic v0.9.3/go.mod h1:nu8jbsbtwK4205EDH/ZUMTKsfYpJA1Q7MKXHfgTihNw= github.com/micro/cli/v2 v2.1.2 h1:43J1lChg/rZCC1rvdqZNFSQDrGT7qfMrtp6/ztpIkEM= github.com/micro/cli/v2 v2.1.2/go.mod h1:EguNh6DAoWKm9nmk+k/Rg0H3lQnDxqzu5x5srOtGtYg= -github.com/micro/mdns v0.3.0 h1:bYycYe+98AXR3s8Nq5qvt6C573uFTDPIYzJemWON0QE= -github.com/micro/mdns v0.3.0/go.mod h1:KJ0dW7KmicXU2BV++qkLlmHYcVv7/hHnbtguSWt9Aoc= -github.com/miekg/dns v1.1.3/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.15/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.27 h1:aEH/kqUzUxGJ/UHcEKdJY+ugH6WEzsEBBSPa8zuy1aM= github.com/miekg/dns v1.1.27/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= @@ -314,7 +309,6 @@ github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRW github.com/namedotcom/go v0.0.0-20180403034216-08470befbe04/go.mod h1:5sN+Lt1CaY4wsPvgQH/jsuJi4XO2ssZbdsIizr4CVC8= github.com/nats-io/jwt v0.3.2 h1:+RB5hMpXUUA2dfxuhBTEkMOrYmM+gKIZYS1KjSostMI= github.com/nats-io/jwt v0.3.2/go.mod h1:/euKqTS1ZD+zzjYrY7pseZrTtWQSjujC7xjPc8wL6eU= -github.com/nats-io/nats-server v1.4.1 h1:Ul1oSOGNV/L8kjr4v6l2f9Yet6WY+LevH1/7cRZ/qyA= github.com/nats-io/nats-server/v2 v2.1.6 h1:qAaHZaS8pRRNQLFaiBA1rq5WynyEGp9DFgmMfoaiXGY= github.com/nats-io/nats-server/v2 v2.1.6/go.mod h1:BL1NOtaBQ5/y97djERRVWNouMW7GT3gxnmbE/eC8u8A= github.com/nats-io/nats.go v1.9.2 h1:oDeERm3NcZVrPpdR/JpGdWHMv3oJ8yY30YwxKq+DU2s= @@ -454,7 +448,6 @@ go.uber.org/zap v1.13.0/go.mod h1:zwrFLgMcdUuIBviXEYEH1YKNaOBnKXsx2IPda5bBwHM= golang.org/x/crypto v0.0.0-20180621125126-a49355c7e3f8/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20181030102418-4d3f4d9ffa16/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20190130090550-b01c7a725664/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190211182817-74369b46fc67/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190219172222-a4c6cb3142f2/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= @@ -465,8 +458,6 @@ golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190829043050-9756ffdc2472/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190927123631-a832865fa7ad/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= -golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d h1:1ZiEyfaQIg3Qh0EoqpwAakHVhecoE5wlSg5GjnafJGw= -golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59 h1:3zb4D3T4G8jdExgVU/95+vQXfpEPiMdCaZgmGVxjNHM= golang.org/x/crypto v0.0.0-20200323165209-0ec3e9974c59/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -530,7 +521,6 @@ golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20181107165924-66b7b1311ac8/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181116152217-5ac8a444bdc5/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= -golang.org/x/sys v0.0.0-20190129075346-302c3dd5f1cc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190209173611-3b5209105503/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190221075227-b4e8571b14e0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= diff --git a/registry/mdns_registry.go b/registry/mdns_registry.go index 7e614840..dfc8e6f0 100644 --- a/registry/mdns_registry.go +++ b/registry/mdns_registry.go @@ -17,7 +17,7 @@ import ( "github.com/google/uuid" "github.com/micro/go-micro/v2/logger" - "github.com/micro/mdns" + "github.com/micro/go-micro/v2/util/mdns" ) var ( diff --git a/registry/mdns_watcher.go b/registry/mdns_watcher.go index 402811b9..e0ef4a48 100644 --- a/registry/mdns_watcher.go +++ b/registry/mdns_watcher.go @@ -4,7 +4,7 @@ import ( "fmt" "strings" - "github.com/micro/mdns" + "github.com/micro/go-micro/v2/util/mdns" ) type mdnsWatcher struct { diff --git a/util/mdns/.gitignore b/util/mdns/.gitignore new file mode 100644 index 00000000..83656241 --- /dev/null +++ b/util/mdns/.gitignore @@ -0,0 +1,23 @@ +# Compiled Object files, Static and Dynamic libs (Shared Objects) +*.o +*.a +*.so + +# Folders +_obj +_test + +# Architecture specific extensions/prefixes +*.[568vq] +[568vq].out + +*.cgo1.go +*.cgo2.c +_cgo_defun.c +_cgo_gotypes.go +_cgo_export.* + +_testmain.go + +*.exe +*.test diff --git a/util/mdns/client.go b/util/mdns/client.go new file mode 100644 index 00000000..176ebac4 --- /dev/null +++ b/util/mdns/client.go @@ -0,0 +1,501 @@ +package mdns + +import ( + "context" + "fmt" + "log" + "net" + "strings" + "sync" + "time" + + "github.com/miekg/dns" + "golang.org/x/net/ipv4" + "golang.org/x/net/ipv6" +) + +// ServiceEntry is returned after we query for a service +type ServiceEntry struct { + Name string + Host string + AddrV4 net.IP + AddrV6 net.IP + Port int + Info string + InfoFields []string + TTL int + Type uint16 + + Addr net.IP // @Deprecated + + hasTXT bool + sent bool +} + +// complete is used to check if we have all the info we need +func (s *ServiceEntry) complete() bool { + return (s.AddrV4 != nil || s.AddrV6 != nil || s.Addr != nil) && s.Port != 0 && s.hasTXT +} + +// QueryParam is used to customize how a Lookup is performed +type QueryParam struct { + Service string // Service to lookup + Domain string // Lookup domain, default "local" + Type uint16 // Lookup type, defaults to dns.TypePTR + Context context.Context // Context + Timeout time.Duration // Lookup timeout, default 1 second. Ignored if Context is provided + Interface *net.Interface // Multicast interface to use + Entries chan<- *ServiceEntry // Entries Channel + WantUnicastResponse bool // Unicast response desired, as per 5.4 in RFC +} + +// DefaultParams is used to return a default set of QueryParam's +func DefaultParams(service string) *QueryParam { + return &QueryParam{ + Service: service, + Domain: "local", + Timeout: time.Second, + Entries: make(chan *ServiceEntry), + WantUnicastResponse: false, // TODO(reddaly): Change this default. + } +} + +// Query looks up a given service, in a domain, waiting at most +// for a timeout before finishing the query. The results are streamed +// to a channel. Sends will not block, so clients should make sure to +// either read or buffer. +func Query(params *QueryParam) error { + // Create a new client + client, err := newClient() + if err != nil { + return err + } + defer client.Close() + + // Set the multicast interface + if params.Interface != nil { + if err := client.setInterface(params.Interface, false); err != nil { + return err + } + } + + // Ensure defaults are set + if params.Domain == "" { + params.Domain = "local" + } + + if params.Context == nil { + if params.Timeout == 0 { + params.Timeout = time.Second + } + params.Context, _ = context.WithTimeout(context.Background(), params.Timeout) + if err != nil { + return err + } + } + + // Run the query + return client.query(params) +} + +// Listen listens indefinitely for multicast updates +func Listen(entries chan<- *ServiceEntry, exit chan struct{}) error { + // Create a new client + client, err := newClient() + if err != nil { + return err + } + defer client.Close() + + client.setInterface(nil, true) + + // Start listening for response packets + msgCh := make(chan *dns.Msg, 32) + + go client.recv(client.ipv4UnicastConn, msgCh) + go client.recv(client.ipv6UnicastConn, msgCh) + go client.recv(client.ipv4MulticastConn, msgCh) + go client.recv(client.ipv6MulticastConn, msgCh) + + ip := make(map[string]*ServiceEntry) + + for { + select { + case <-exit: + return nil + case <-client.closedCh: + return nil + case m := <-msgCh: + e := messageToEntry(m, ip) + if e == nil { + continue + } + + // Check if this entry is complete + if e.complete() { + if e.sent { + continue + } + e.sent = true + entries <- e + ip = make(map[string]*ServiceEntry) + } else { + // Fire off a node specific query + m := new(dns.Msg) + m.SetQuestion(e.Name, dns.TypePTR) + m.RecursionDesired = false + if err := client.sendQuery(m); err != nil { + log.Printf("[ERR] mdns: Failed to query instance %s: %v", e.Name, err) + } + } + } + } + + return nil +} + +// Lookup is the same as Query, however it uses all the default parameters +func Lookup(service string, entries chan<- *ServiceEntry) error { + params := DefaultParams(service) + params.Entries = entries + return Query(params) +} + +// Client provides a query interface that can be used to +// search for service providers using mDNS +type client struct { + ipv4UnicastConn *net.UDPConn + ipv6UnicastConn *net.UDPConn + + ipv4MulticastConn *net.UDPConn + ipv6MulticastConn *net.UDPConn + + closed bool + closedCh chan struct{} // TODO(reddaly): This doesn't appear to be used. + closeLock sync.Mutex +} + +// NewClient creates a new mdns Client that can be used to query +// for records +func newClient() (*client, error) { + // TODO(reddaly): At least attempt to bind to the port required in the spec. + // Create a IPv4 listener + uconn4, err4 := net.ListenUDP("udp4", &net.UDPAddr{IP: net.IPv4zero, Port: 0}) + uconn6, err6 := net.ListenUDP("udp6", &net.UDPAddr{IP: net.IPv6zero, Port: 0}) + if err4 != nil && err6 != nil { + log.Printf("[ERR] mdns: Failed to bind to udp port: %v %v", err4, err6) + } + + if uconn4 == nil && uconn6 == nil { + return nil, fmt.Errorf("failed to bind to any unicast udp port") + } + + if uconn4 == nil { + uconn4 = &net.UDPConn{} + } + + if uconn6 == nil { + uconn6 = &net.UDPConn{} + } + + mconn4, err4 := net.ListenUDP("udp4", mdnsWildcardAddrIPv4) + mconn6, err6 := net.ListenUDP("udp6", mdnsWildcardAddrIPv6) + if err4 != nil && err6 != nil { + log.Printf("[ERR] mdns: Failed to bind to udp port: %v %v", err4, err6) + } + + if mconn4 == nil && mconn6 == nil { + return nil, fmt.Errorf("failed to bind to any multicast udp port") + } + + if mconn4 == nil { + mconn4 = &net.UDPConn{} + } + + if mconn6 == nil { + mconn6 = &net.UDPConn{} + } + + p1 := ipv4.NewPacketConn(mconn4) + p2 := ipv6.NewPacketConn(mconn6) + p1.SetMulticastLoopback(true) + p2.SetMulticastLoopback(true) + + ifaces, err := net.Interfaces() + if err != nil { + return nil, err + } + + var errCount1, errCount2 int + + for _, iface := range ifaces { + if err := p1.JoinGroup(&iface, &net.UDPAddr{IP: mdnsGroupIPv4}); err != nil { + errCount1++ + } + if err := p2.JoinGroup(&iface, &net.UDPAddr{IP: mdnsGroupIPv6}); err != nil { + errCount2++ + } + } + + if len(ifaces) == errCount1 && len(ifaces) == errCount2 { + return nil, fmt.Errorf("Failed to join multicast group on all interfaces!") + } + + c := &client{ + ipv4MulticastConn: mconn4, + ipv6MulticastConn: mconn6, + ipv4UnicastConn: uconn4, + ipv6UnicastConn: uconn6, + closedCh: make(chan struct{}), + } + return c, nil +} + +// Close is used to cleanup the client +func (c *client) Close() error { + c.closeLock.Lock() + defer c.closeLock.Unlock() + + if c.closed { + return nil + } + c.closed = true + + close(c.closedCh) + + if c.ipv4UnicastConn != nil { + c.ipv4UnicastConn.Close() + } + if c.ipv6UnicastConn != nil { + c.ipv6UnicastConn.Close() + } + if c.ipv4MulticastConn != nil { + c.ipv4MulticastConn.Close() + } + if c.ipv6MulticastConn != nil { + c.ipv6MulticastConn.Close() + } + + return nil +} + +// setInterface is used to set the query interface, uses sytem +// default if not provided +func (c *client) setInterface(iface *net.Interface, loopback bool) error { + p := ipv4.NewPacketConn(c.ipv4UnicastConn) + if err := p.JoinGroup(iface, &net.UDPAddr{IP: mdnsGroupIPv4}); err != nil { + return err + } + p2 := ipv6.NewPacketConn(c.ipv6UnicastConn) + if err := p2.JoinGroup(iface, &net.UDPAddr{IP: mdnsGroupIPv6}); err != nil { + return err + } + p = ipv4.NewPacketConn(c.ipv4MulticastConn) + if err := p.JoinGroup(iface, &net.UDPAddr{IP: mdnsGroupIPv4}); err != nil { + return err + } + p2 = ipv6.NewPacketConn(c.ipv6MulticastConn) + if err := p2.JoinGroup(iface, &net.UDPAddr{IP: mdnsGroupIPv6}); err != nil { + return err + } + + if loopback { + p.SetMulticastLoopback(true) + p2.SetMulticastLoopback(true) + } + + return nil +} + +// query is used to perform a lookup and stream results +func (c *client) query(params *QueryParam) error { + // Create the service name + serviceAddr := fmt.Sprintf("%s.%s.", trimDot(params.Service), trimDot(params.Domain)) + + // Start listening for response packets + msgCh := make(chan *dns.Msg, 32) + go c.recv(c.ipv4UnicastConn, msgCh) + go c.recv(c.ipv6UnicastConn, msgCh) + go c.recv(c.ipv4MulticastConn, msgCh) + go c.recv(c.ipv6MulticastConn, msgCh) + + // Send the query + m := new(dns.Msg) + if params.Type == dns.TypeNone { + m.SetQuestion(serviceAddr, dns.TypePTR) + } else { + m.SetQuestion(serviceAddr, params.Type) + } + // RFC 6762, section 18.12. Repurposing of Top Bit of qclass in Question + // Section + // + // In the Question Section of a Multicast DNS query, the top bit of the qclass + // field is used to indicate that unicast responses are preferred for this + // particular question. (See Section 5.4.) + if params.WantUnicastResponse { + m.Question[0].Qclass |= 1 << 15 + } + m.RecursionDesired = false + if err := c.sendQuery(m); err != nil { + return err + } + + // Map the in-progress responses + inprogress := make(map[string]*ServiceEntry) + + for { + select { + case resp := <-msgCh: + inp := messageToEntry(resp, inprogress) + if inp == nil { + continue + } + + // Check if this entry is complete + if inp.complete() { + if inp.sent { + continue + } + inp.sent = true + select { + case params.Entries <- inp: + case <-params.Context.Done(): + return nil + } + } else { + // Fire off a node specific query + m := new(dns.Msg) + m.SetQuestion(inp.Name, inp.Type) + m.RecursionDesired = false + if err := c.sendQuery(m); err != nil { + log.Printf("[ERR] mdns: Failed to query instance %s: %v", inp.Name, err) + } + } + case <-params.Context.Done(): + return nil + } + } +} + +// sendQuery is used to multicast a query out +func (c *client) sendQuery(q *dns.Msg) error { + buf, err := q.Pack() + if err != nil { + return err + } + if c.ipv4UnicastConn != nil { + c.ipv4UnicastConn.WriteToUDP(buf, ipv4Addr) + } + if c.ipv6UnicastConn != nil { + c.ipv6UnicastConn.WriteToUDP(buf, ipv6Addr) + } + return nil +} + +// recv is used to receive until we get a shutdown +func (c *client) recv(l *net.UDPConn, msgCh chan *dns.Msg) { + if l == nil { + return + } + buf := make([]byte, 65536) + for { + c.closeLock.Lock() + if c.closed { + c.closeLock.Unlock() + return + } + c.closeLock.Unlock() + n, err := l.Read(buf) + if err != nil { + continue + } + msg := new(dns.Msg) + if err := msg.Unpack(buf[:n]); err != nil { + continue + } + select { + case msgCh <- msg: + case <-c.closedCh: + return + } + } +} + +// ensureName is used to ensure the named node is in progress +func ensureName(inprogress map[string]*ServiceEntry, name string, typ uint16) *ServiceEntry { + if inp, ok := inprogress[name]; ok { + return inp + } + inp := &ServiceEntry{ + Name: name, + Type: typ, + } + inprogress[name] = inp + return inp +} + +// alias is used to setup an alias between two entries +func alias(inprogress map[string]*ServiceEntry, src, dst string, typ uint16) { + srcEntry := ensureName(inprogress, src, typ) + inprogress[dst] = srcEntry +} + +func messageToEntry(m *dns.Msg, inprogress map[string]*ServiceEntry) *ServiceEntry { + var inp *ServiceEntry + + for _, answer := range append(m.Answer, m.Extra...) { + // TODO(reddaly): Check that response corresponds to serviceAddr? + switch rr := answer.(type) { + case *dns.PTR: + // Create new entry for this + inp = ensureName(inprogress, rr.Ptr, rr.Hdr.Rrtype) + if inp.complete() { + continue + } + case *dns.SRV: + // Check for a target mismatch + if rr.Target != rr.Hdr.Name { + alias(inprogress, rr.Hdr.Name, rr.Target, rr.Hdr.Rrtype) + } + + // Get the port + inp = ensureName(inprogress, rr.Hdr.Name, rr.Hdr.Rrtype) + if inp.complete() { + continue + } + inp.Host = rr.Target + inp.Port = int(rr.Port) + case *dns.TXT: + // Pull out the txt + inp = ensureName(inprogress, rr.Hdr.Name, rr.Hdr.Rrtype) + if inp.complete() { + continue + } + inp.Info = strings.Join(rr.Txt, "|") + inp.InfoFields = rr.Txt + inp.hasTXT = true + case *dns.A: + // Pull out the IP + inp = ensureName(inprogress, rr.Hdr.Name, rr.Hdr.Rrtype) + if inp.complete() { + continue + } + inp.Addr = rr.A // @Deprecated + inp.AddrV4 = rr.A + case *dns.AAAA: + // Pull out the IP + inp = ensureName(inprogress, rr.Hdr.Name, rr.Hdr.Rrtype) + if inp.complete() { + continue + } + inp.Addr = rr.AAAA // @Deprecated + inp.AddrV6 = rr.AAAA + } + + if inp != nil { + inp.TTL = int(answer.Header().Ttl) + } + } + + return inp +} diff --git a/util/mdns/dns_sd.go b/util/mdns/dns_sd.go new file mode 100644 index 00000000..18444c34 --- /dev/null +++ b/util/mdns/dns_sd.go @@ -0,0 +1,84 @@ +package mdns + +import "github.com/miekg/dns" + +// DNSSDService is a service that complies with the DNS-SD (RFC 6762) and MDNS +// (RFC 6762) specs for local, multicast-DNS-based discovery. +// +// DNSSDService implements the Zone interface and wraps an MDNSService instance. +// To deploy an mDNS service that is compliant with DNS-SD, it's recommended to +// register only the wrapped instance with the server. +// +// Example usage: +// service := &mdns.DNSSDService{ +// MDNSService: &mdns.MDNSService{ +// Instance: "My Foobar Service", +// Service: "_foobar._tcp", +// Port: 8000, +// } +// } +// server, err := mdns.NewServer(&mdns.Config{Zone: service}) +// if err != nil { +// log.Fatalf("Error creating server: %v", err) +// } +// defer server.Shutdown() +type DNSSDService struct { + MDNSService *MDNSService +} + +// Records returns DNS records in response to a DNS question. +// +// This function returns the DNS response of the underlying MDNSService +// instance. It also returns a PTR record for a request for " +// _services._dns-sd._udp.", as described in section 9 of RFC 6763 +// ("Service Type Enumeration"), to allow browsing of the underlying MDNSService +// instance. +func (s *DNSSDService) Records(q dns.Question) []dns.RR { + var recs []dns.RR + if q.Name == "_services._dns-sd._udp."+s.MDNSService.Domain+"." { + recs = s.dnssdMetaQueryRecords(q) + } + return append(recs, s.MDNSService.Records(q)...) +} + +// dnssdMetaQueryRecords returns the DNS records in response to a "meta-query" +// issued to browse for DNS-SD services, as per section 9. of RFC6763. +// +// A meta-query has a name of the form "_services._dns-sd._udp." where +// Domain is a fully-qualified domain, such as "local." +func (s *DNSSDService) dnssdMetaQueryRecords(q dns.Question) []dns.RR { + // Intended behavior, as described in the RFC: + // ...it may be useful for network administrators to find the list of + // advertised service types on the network, even if those Service Names + // are just opaque identifiers and not particularly informative in + // isolation. + // + // For this purpose, a special meta-query is defined. A DNS query for PTR + // records with the name "_services._dns-sd._udp." yields a set of + // PTR records, where the rdata of each PTR record is the two-abel + // name, plus the same domain, e.g., "_http._tcp.". + // Including the domain in the PTR rdata allows for slightly better name + // compression in Unicast DNS responses, but only the first two labels are + // relevant for the purposes of service type enumeration. These two-label + // service types can then be used to construct subsequent Service Instance + // Enumeration PTR queries, in this or others, to discover + // instances of that service type. + return []dns.RR{ + &dns.PTR{ + Hdr: dns.RR_Header{ + Name: q.Name, + Rrtype: dns.TypePTR, + Class: dns.ClassINET, + Ttl: defaultTTL, + }, + Ptr: s.MDNSService.serviceAddr, + }, + } +} + +// Announcement returns DNS records that should be broadcast during the initial +// availability of the service, as described in section 8.3 of RFC 6762. +// TODO(reddaly): Add this when Announcement is added to the mdns.Zone interface. +//func (s *DNSSDService) Announcement() []dns.RR { +// return s.MDNSService.Announcement() +//} diff --git a/util/mdns/dns_sd_test.go b/util/mdns/dns_sd_test.go new file mode 100644 index 00000000..d973fd7e --- /dev/null +++ b/util/mdns/dns_sd_test.go @@ -0,0 +1,68 @@ +package mdns + +import ( + "reflect" + "testing" +) +import "github.com/miekg/dns" + +type mockMDNSService struct{} + +func (s *mockMDNSService) Records(q dns.Question) []dns.RR { + return []dns.RR{ + &dns.PTR{ + Hdr: dns.RR_Header{ + Name: "fakerecord", + Rrtype: dns.TypePTR, + Class: dns.ClassINET, + Ttl: 42, + }, + Ptr: "fake.local.", + }, + } +} + +func (s *mockMDNSService) Announcement() []dns.RR { + return []dns.RR{ + &dns.PTR{ + Hdr: dns.RR_Header{ + Name: "fakeannounce", + Rrtype: dns.TypePTR, + Class: dns.ClassINET, + Ttl: 42, + }, + Ptr: "fake.local.", + }, + } +} + +func TestDNSSDServiceRecords(t *testing.T) { + s := &DNSSDService{ + MDNSService: &MDNSService{ + serviceAddr: "_foobar._tcp.local.", + Domain: "local", + }, + } + q := dns.Question{ + Name: "_services._dns-sd._udp.local.", + Qtype: dns.TypePTR, + Qclass: dns.ClassINET, + } + recs := s.Records(q) + if got, want := len(recs), 1; got != want { + t.Fatalf("s.Records(%v) returned %v records, want %v", q, got, want) + } + + want := dns.RR(&dns.PTR{ + Hdr: dns.RR_Header{ + Name: "_services._dns-sd._udp.local.", + Rrtype: dns.TypePTR, + Class: dns.ClassINET, + Ttl: defaultTTL, + }, + Ptr: "_foobar._tcp.local.", + }) + if got := recs[0]; !reflect.DeepEqual(got, want) { + t.Errorf("s.Records()[0] = %v, want %v", got, want) + } +} diff --git a/util/mdns/server.go b/util/mdns/server.go new file mode 100644 index 00000000..909b39c5 --- /dev/null +++ b/util/mdns/server.go @@ -0,0 +1,476 @@ +package mdns + +import ( + "fmt" + "log" + "math/rand" + "net" + "sync" + "sync/atomic" + "time" + + "github.com/miekg/dns" + "golang.org/x/net/ipv4" + "golang.org/x/net/ipv6" +) + +var ( + mdnsGroupIPv4 = net.ParseIP("224.0.0.251") + mdnsGroupIPv6 = net.ParseIP("ff02::fb") + + // mDNS wildcard addresses + mdnsWildcardAddrIPv4 = &net.UDPAddr{ + IP: net.ParseIP("224.0.0.0"), + Port: 5353, + } + mdnsWildcardAddrIPv6 = &net.UDPAddr{ + IP: net.ParseIP("ff02::"), + Port: 5353, + } + + // mDNS endpoint addresses + ipv4Addr = &net.UDPAddr{ + IP: mdnsGroupIPv4, + Port: 5353, + } + ipv6Addr = &net.UDPAddr{ + IP: mdnsGroupIPv6, + Port: 5353, + } +) + +// Config is used to configure the mDNS server +type Config struct { + // Zone must be provided to support responding to queries + Zone Zone + + // Iface if provided binds the multicast listener to the given + // interface. If not provided, the system default multicase interface + // is used. + Iface *net.Interface + + // Port If it is not 0, replace the port 5353 with this port number. + Port int +} + +// mDNS server is used to listen for mDNS queries and respond if we +// have a matching local record +type Server struct { + config *Config + + ipv4List *net.UDPConn + ipv6List *net.UDPConn + + shutdown bool + shutdownCh chan struct{} + shutdownLock sync.Mutex + wg sync.WaitGroup +} + +// NewServer is used to create a new mDNS server from a config +func NewServer(config *Config) (*Server, error) { + setCustomPort(config.Port) + + // Create the listeners + // Create wildcard connections (because :5353 can be already taken by other apps) + ipv4List, _ := net.ListenUDP("udp4", mdnsWildcardAddrIPv4) + ipv6List, _ := net.ListenUDP("udp6", mdnsWildcardAddrIPv6) + if ipv4List == nil && ipv6List == nil { + return nil, fmt.Errorf("[ERR] mdns: Failed to bind to any udp port!") + } + + if ipv4List == nil { + ipv4List = &net.UDPConn{} + } + if ipv6List == nil { + ipv6List = &net.UDPConn{} + } + + // Join multicast groups to receive announcements + p1 := ipv4.NewPacketConn(ipv4List) + p2 := ipv6.NewPacketConn(ipv6List) + p1.SetMulticastLoopback(true) + p2.SetMulticastLoopback(true) + + if config.Iface != nil { + if err := p1.JoinGroup(config.Iface, &net.UDPAddr{IP: mdnsGroupIPv4}); err != nil { + return nil, err + } + if err := p2.JoinGroup(config.Iface, &net.UDPAddr{IP: mdnsGroupIPv6}); err != nil { + return nil, err + } + } else { + ifaces, err := net.Interfaces() + if err != nil { + return nil, err + } + errCount1, errCount2 := 0, 0 + for _, iface := range ifaces { + if err := p1.JoinGroup(&iface, &net.UDPAddr{IP: mdnsGroupIPv4}); err != nil { + errCount1++ + } + if err := p2.JoinGroup(&iface, &net.UDPAddr{IP: mdnsGroupIPv6}); err != nil { + errCount2++ + } + } + if len(ifaces) == errCount1 && len(ifaces) == errCount2 { + return nil, fmt.Errorf("Failed to join multicast group on all interfaces!") + } + } + + s := &Server{ + config: config, + ipv4List: ipv4List, + ipv6List: ipv6List, + shutdownCh: make(chan struct{}), + } + + go s.recv(s.ipv4List) + go s.recv(s.ipv6List) + + s.wg.Add(1) + go s.probe() + + return s, nil +} + +// Shutdown is used to shutdown the listener +func (s *Server) Shutdown() error { + s.shutdownLock.Lock() + defer s.shutdownLock.Unlock() + + if s.shutdown { + return nil + } + + s.shutdown = true + close(s.shutdownCh) + s.unregister() + + if s.ipv4List != nil { + s.ipv4List.Close() + } + if s.ipv6List != nil { + s.ipv6List.Close() + } + + s.wg.Wait() + return nil +} + +// recv is a long running routine to receive packets from an interface +func (s *Server) recv(c *net.UDPConn) { + if c == nil { + return + } + buf := make([]byte, 65536) + for { + s.shutdownLock.Lock() + if s.shutdown { + s.shutdownLock.Unlock() + return + } + s.shutdownLock.Unlock() + n, from, err := c.ReadFrom(buf) + if err != nil { + continue + } + if err := s.parsePacket(buf[:n], from); err != nil { + log.Printf("[ERR] mdns: Failed to handle query: %v", err) + } + } +} + +// parsePacket is used to parse an incoming packet +func (s *Server) parsePacket(packet []byte, from net.Addr) error { + var msg dns.Msg + if err := msg.Unpack(packet); err != nil { + log.Printf("[ERR] mdns: Failed to unpack packet: %v", err) + return err + } + // TODO: This is a bit of a hack + // We decided to ignore some mDNS answers for the time being + // See: https://tools.ietf.org/html/rfc6762#section-7.2 + msg.Truncated = false + return s.handleQuery(&msg, from) +} + +// handleQuery is used to handle an incoming query +func (s *Server) handleQuery(query *dns.Msg, from net.Addr) error { + if query.Opcode != dns.OpcodeQuery { + // "In both multicast query and multicast response messages, the OPCODE MUST + // be zero on transmission (only standard queries are currently supported + // over multicast). Multicast DNS messages received with an OPCODE other + // than zero MUST be silently ignored." Note: OpcodeQuery == 0 + return fmt.Errorf("mdns: received query with non-zero Opcode %v: %v", query.Opcode, *query) + } + if query.Rcode != 0 { + // "In both multicast query and multicast response messages, the Response + // Code MUST be zero on transmission. Multicast DNS messages received with + // non-zero Response Codes MUST be silently ignored." + return fmt.Errorf("mdns: received query with non-zero Rcode %v: %v", query.Rcode, *query) + } + + // TODO(reddaly): Handle "TC (Truncated) Bit": + // In query messages, if the TC bit is set, it means that additional + // Known-Answer records may be following shortly. A responder SHOULD + // record this fact, and wait for those additional Known-Answer records, + // before deciding whether to respond. If the TC bit is clear, it means + // that the querying host has no additional Known Answers. + if query.Truncated { + return fmt.Errorf("[ERR] mdns: support for DNS requests with high truncated bit not implemented: %v", *query) + } + + var unicastAnswer, multicastAnswer []dns.RR + + // Handle each question + for _, q := range query.Question { + mrecs, urecs := s.handleQuestion(q) + multicastAnswer = append(multicastAnswer, mrecs...) + unicastAnswer = append(unicastAnswer, urecs...) + } + + // See section 18 of RFC 6762 for rules about DNS headers. + resp := func(unicast bool) *dns.Msg { + // 18.1: ID (Query Identifier) + // 0 for multicast response, query.Id for unicast response + id := uint16(0) + if unicast { + id = query.Id + } + + var answer []dns.RR + if unicast { + answer = unicastAnswer + } else { + answer = multicastAnswer + } + if len(answer) == 0 { + return nil + } + + return &dns.Msg{ + MsgHdr: dns.MsgHdr{ + Id: id, + + // 18.2: QR (Query/Response) Bit - must be set to 1 in response. + Response: true, + + // 18.3: OPCODE - must be zero in response (OpcodeQuery == 0) + Opcode: dns.OpcodeQuery, + + // 18.4: AA (Authoritative Answer) Bit - must be set to 1 + Authoritative: true, + + // The following fields must all be set to 0: + // 18.5: TC (TRUNCATED) Bit + // 18.6: RD (Recursion Desired) Bit + // 18.7: RA (Recursion Available) Bit + // 18.8: Z (Zero) Bit + // 18.9: AD (Authentic Data) Bit + // 18.10: CD (Checking Disabled) Bit + // 18.11: RCODE (Response Code) + }, + // 18.12 pertains to questions (handled by handleQuestion) + // 18.13 pertains to resource records (handled by handleQuestion) + + // 18.14: Name Compression - responses should be compressed (though see + // caveats in the RFC), so set the Compress bit (part of the dns library + // API, not part of the DNS packet) to true. + Compress: true, + + Answer: answer, + } + } + + if mresp := resp(false); mresp != nil { + if err := s.sendResponse(mresp, from); err != nil { + return fmt.Errorf("mdns: error sending multicast response: %v", err) + } + } + if uresp := resp(true); uresp != nil { + if err := s.sendResponse(uresp, from); err != nil { + return fmt.Errorf("mdns: error sending unicast response: %v", err) + } + } + return nil +} + +// handleQuestion is used to handle an incoming question +// +// The response to a question may be transmitted over multicast, unicast, or +// both. The return values are DNS records for each transmission type. +func (s *Server) handleQuestion(q dns.Question) (multicastRecs, unicastRecs []dns.RR) { + records := s.config.Zone.Records(q) + + if len(records) == 0 { + return nil, nil + } + + // Handle unicast and multicast responses. + // TODO(reddaly): The decision about sending over unicast vs. multicast is not + // yet fully compliant with RFC 6762. For example, the unicast bit should be + // ignored if the records in question are close to TTL expiration. For now, + // we just use the unicast bit to make the decision, as per the spec: + // RFC 6762, section 18.12. Repurposing of Top Bit of qclass in Question + // Section + // + // In the Question Section of a Multicast DNS query, the top bit of the + // qclass field is used to indicate that unicast responses are preferred + // for this particular question. (See Section 5.4.) + if q.Qclass&(1<<15) != 0 { + return nil, records + } + return records, nil +} + +func (s *Server) probe() { + defer s.wg.Done() + + sd, ok := s.config.Zone.(*MDNSService) + if !ok { + return + } + + name := fmt.Sprintf("%s.%s.%s.", sd.Instance, trimDot(sd.Service), trimDot(sd.Domain)) + + q := new(dns.Msg) + q.SetQuestion(name, dns.TypePTR) + q.RecursionDesired = false + + srv := &dns.SRV{ + Hdr: dns.RR_Header{ + Name: name, + Rrtype: dns.TypeSRV, + Class: dns.ClassINET, + Ttl: defaultTTL, + }, + Priority: 0, + Weight: 0, + Port: uint16(sd.Port), + Target: sd.HostName, + } + txt := &dns.TXT{ + Hdr: dns.RR_Header{ + Name: name, + Rrtype: dns.TypeTXT, + Class: dns.ClassINET, + Ttl: defaultTTL, + }, + Txt: sd.TXT, + } + q.Ns = []dns.RR{srv, txt} + + randomizer := rand.New(rand.NewSource(time.Now().UnixNano())) + + for i := 0; i < 3; i++ { + if err := s.SendMulticast(q); err != nil { + log.Println("[ERR] mdns: failed to send probe:", err.Error()) + } + time.Sleep(time.Duration(randomizer.Intn(250)) * time.Millisecond) + } + + resp := new(dns.Msg) + resp.MsgHdr.Response = true + + // set for query + q.SetQuestion(name, dns.TypeANY) + + resp.Answer = append(resp.Answer, s.config.Zone.Records(q.Question[0])...) + + // reset + q.SetQuestion(name, dns.TypePTR) + + // From RFC6762 + // The Multicast DNS responder MUST send at least two unsolicited + // responses, one second apart. To provide increased robustness against + // packet loss, a responder MAY send up to eight unsolicited responses, + // provided that the interval between unsolicited responses increases by + // at least a factor of two with every response sent. + timeout := 1 * time.Second + timer := time.NewTimer(timeout) + for i := 0; i < 3; i++ { + if err := s.SendMulticast(resp); err != nil { + log.Println("[ERR] mdns: failed to send announcement:", err.Error()) + } + select { + case <-timer.C: + timeout *= 2 + timer.Reset(timeout) + case <-s.shutdownCh: + timer.Stop() + return + } + } +} + +// multicastResponse us used to send a multicast response packet +func (s *Server) SendMulticast(msg *dns.Msg) error { + buf, err := msg.Pack() + if err != nil { + return err + } + if s.ipv4List != nil { + s.ipv4List.WriteToUDP(buf, ipv4Addr) + } + if s.ipv6List != nil { + s.ipv6List.WriteToUDP(buf, ipv6Addr) + } + return nil +} + +// sendResponse is used to send a response packet +func (s *Server) sendResponse(resp *dns.Msg, from net.Addr) error { + // TODO(reddaly): Respect the unicast argument, and allow sending responses + // over multicast. + buf, err := resp.Pack() + if err != nil { + return err + } + + // Determine the socket to send from + addr := from.(*net.UDPAddr) + if addr.IP.To4() != nil { + _, err = s.ipv4List.WriteToUDP(buf, addr) + return err + } else { + _, err = s.ipv6List.WriteToUDP(buf, addr) + return err + } +} + +func (s *Server) unregister() error { + sd, ok := s.config.Zone.(*MDNSService) + if !ok { + return nil + } + + atomic.StoreUint32(&sd.TTL, 0) + name := fmt.Sprintf("%s.%s.%s.", sd.Instance, trimDot(sd.Service), trimDot(sd.Domain)) + + q := new(dns.Msg) + q.SetQuestion(name, dns.TypeANY) + + resp := new(dns.Msg) + resp.MsgHdr.Response = true + resp.Answer = append(resp.Answer, s.config.Zone.Records(q.Question[0])...) + + return s.SendMulticast(resp) +} + +func setCustomPort(port int) { + if port != 0 { + if mdnsWildcardAddrIPv4.Port != port { + mdnsWildcardAddrIPv4.Port = port + } + if mdnsWildcardAddrIPv6.Port != port { + mdnsWildcardAddrIPv6.Port = port + } + if ipv4Addr.Port != port { + ipv4Addr.Port = port + } + if ipv6Addr.Port != port { + ipv6Addr.Port = port + } + } +} diff --git a/util/mdns/server_test.go b/util/mdns/server_test.go new file mode 100644 index 00000000..6fb00fa2 --- /dev/null +++ b/util/mdns/server_test.go @@ -0,0 +1,61 @@ +package mdns + +import ( + "testing" + "time" +) + +func TestServer_StartStop(t *testing.T) { + s := makeService(t) + serv, err := NewServer(&Config{Zone: s}) + if err != nil { + t.Fatalf("err: %v", err) + } + defer serv.Shutdown() +} + +func TestServer_Lookup(t *testing.T) { + serv, err := NewServer(&Config{Zone: makeServiceWithServiceName(t, "_foobar._tcp")}) + if err != nil { + t.Fatalf("err: %v", err) + } + defer serv.Shutdown() + + entries := make(chan *ServiceEntry, 1) + found := false + doneCh := make(chan struct{}) + go func() { + select { + case e := <-entries: + if e.Name != "hostname._foobar._tcp.local." { + t.Fatalf("bad: %v", e) + } + if e.Port != 80 { + t.Fatalf("bad: %v", e) + } + if e.Info != "Local web server" { + t.Fatalf("bad: %v", e) + } + found = true + + case <-time.After(80 * time.Millisecond): + t.Fatalf("timeout") + } + close(doneCh) + }() + + params := &QueryParam{ + Service: "_foobar._tcp", + Domain: "local", + Timeout: 50 * time.Millisecond, + Entries: entries, + } + err = Query(params) + if err != nil { + t.Fatalf("err: %v", err) + } + <-doneCh + if !found { + t.Fatalf("record not found") + } +} diff --git a/util/mdns/zone.go b/util/mdns/zone.go new file mode 100644 index 00000000..abbab4bd --- /dev/null +++ b/util/mdns/zone.go @@ -0,0 +1,309 @@ +package mdns + +import ( + "fmt" + "net" + "os" + "strings" + "sync/atomic" + + "github.com/miekg/dns" +) + +const ( + // defaultTTL is the default TTL value in returned DNS records in seconds. + defaultTTL = 120 +) + +// Zone is the interface used to integrate with the server and +// to serve records dynamically +type Zone interface { + // Records returns DNS records in response to a DNS question. + Records(q dns.Question) []dns.RR +} + +// MDNSService is used to export a named service by implementing a Zone +type MDNSService struct { + Instance string // Instance name (e.g. "hostService name") + Service string // Service name (e.g. "_http._tcp.") + Domain string // If blank, assumes "local" + HostName string // Host machine DNS name (e.g. "mymachine.net.") + Port int // Service Port + IPs []net.IP // IP addresses for the service's host + TXT []string // Service TXT records + TTL uint32 + serviceAddr string // Fully qualified service address + instanceAddr string // Fully qualified instance address + enumAddr string // _services._dns-sd._udp. +} + +// validateFQDN returns an error if the passed string is not a fully qualified +// hdomain name (more specifically, a hostname). +func validateFQDN(s string) error { + if len(s) == 0 { + return fmt.Errorf("FQDN must not be blank") + } + if s[len(s)-1] != '.' { + return fmt.Errorf("FQDN must end in period: %s", s) + } + // TODO(reddaly): Perform full validation. + + return nil +} + +// NewMDNSService returns a new instance of MDNSService. +// +// If domain, hostName, or ips is set to the zero value, then a default value +// will be inferred from the operating system. +// +// TODO(reddaly): This interface may need to change to account for "unique +// record" conflict rules of the mDNS protocol. Upon startup, the server should +// check to ensure that the instance name does not conflict with other instance +// names, and, if required, select a new name. There may also be conflicting +// hostName A/AAAA records. +func NewMDNSService(instance, service, domain, hostName string, port int, ips []net.IP, txt []string) (*MDNSService, error) { + // Sanity check inputs + if instance == "" { + return nil, fmt.Errorf("missing service instance name") + } + if service == "" { + return nil, fmt.Errorf("missing service name") + } + if port == 0 { + return nil, fmt.Errorf("missing service port") + } + + // Set default domain + if domain == "" { + domain = "local." + } + if err := validateFQDN(domain); err != nil { + return nil, fmt.Errorf("domain %q is not a fully-qualified domain name: %v", domain, err) + } + + // Get host information if no host is specified. + if hostName == "" { + var err error + hostName, err = os.Hostname() + if err != nil { + return nil, fmt.Errorf("could not determine host: %v", err) + } + hostName = fmt.Sprintf("%s.", hostName) + } + if err := validateFQDN(hostName); err != nil { + return nil, fmt.Errorf("hostName %q is not a fully-qualified domain name: %v", hostName, err) + } + + if len(ips) == 0 { + var err error + ips, err = net.LookupIP(trimDot(hostName)) + if err != nil { + // Try appending the host domain suffix and lookup again + // (required for Linux-based hosts) + tmpHostName := fmt.Sprintf("%s%s", hostName, domain) + + ips, err = net.LookupIP(trimDot(tmpHostName)) + + if err != nil { + return nil, fmt.Errorf("could not determine host IP addresses for %s", hostName) + } + } + } + for _, ip := range ips { + if ip.To4() == nil && ip.To16() == nil { + return nil, fmt.Errorf("invalid IP address in IPs list: %v", ip) + } + } + + return &MDNSService{ + Instance: instance, + Service: service, + Domain: domain, + HostName: hostName, + Port: port, + IPs: ips, + TXT: txt, + TTL: defaultTTL, + serviceAddr: fmt.Sprintf("%s.%s.", trimDot(service), trimDot(domain)), + instanceAddr: fmt.Sprintf("%s.%s.%s.", instance, trimDot(service), trimDot(domain)), + enumAddr: fmt.Sprintf("_services._dns-sd._udp.%s.", trimDot(domain)), + }, nil +} + +// trimDot is used to trim the dots from the start or end of a string +func trimDot(s string) string { + return strings.Trim(s, ".") +} + +// Records returns DNS records in response to a DNS question. +func (m *MDNSService) Records(q dns.Question) []dns.RR { + switch q.Name { + case m.enumAddr: + return m.serviceEnum(q) + case m.serviceAddr: + return m.serviceRecords(q) + case m.instanceAddr: + return m.instanceRecords(q) + case m.HostName: + if q.Qtype == dns.TypeA || q.Qtype == dns.TypeAAAA { + return m.instanceRecords(q) + } + fallthrough + default: + return nil + } +} + +func (m *MDNSService) serviceEnum(q dns.Question) []dns.RR { + switch q.Qtype { + case dns.TypeANY: + fallthrough + case dns.TypePTR: + rr := &dns.PTR{ + Hdr: dns.RR_Header{ + Name: q.Name, + Rrtype: dns.TypePTR, + Class: dns.ClassINET, + Ttl: atomic.LoadUint32(&m.TTL), + }, + Ptr: m.serviceAddr, + } + return []dns.RR{rr} + default: + return nil + } +} + +// serviceRecords is called when the query matches the service name +func (m *MDNSService) serviceRecords(q dns.Question) []dns.RR { + switch q.Qtype { + case dns.TypeANY: + fallthrough + case dns.TypePTR: + // Build a PTR response for the service + rr := &dns.PTR{ + Hdr: dns.RR_Header{ + Name: q.Name, + Rrtype: dns.TypePTR, + Class: dns.ClassINET, + Ttl: atomic.LoadUint32(&m.TTL), + }, + Ptr: m.instanceAddr, + } + servRec := []dns.RR{rr} + + // Get the instance records + instRecs := m.instanceRecords(dns.Question{ + Name: m.instanceAddr, + Qtype: dns.TypeANY, + }) + + // Return the service record with the instance records + return append(servRec, instRecs...) + default: + return nil + } +} + +// serviceRecords is called when the query matches the instance name +func (m *MDNSService) instanceRecords(q dns.Question) []dns.RR { + switch q.Qtype { + case dns.TypeANY: + // Get the SRV, which includes A and AAAA + recs := m.instanceRecords(dns.Question{ + Name: m.instanceAddr, + Qtype: dns.TypeSRV, + }) + + // Add the TXT record + recs = append(recs, m.instanceRecords(dns.Question{ + Name: m.instanceAddr, + Qtype: dns.TypeTXT, + })...) + return recs + + case dns.TypeA: + var rr []dns.RR + for _, ip := range m.IPs { + if ip4 := ip.To4(); ip4 != nil { + rr = append(rr, &dns.A{ + Hdr: dns.RR_Header{ + Name: m.HostName, + Rrtype: dns.TypeA, + Class: dns.ClassINET, + Ttl: atomic.LoadUint32(&m.TTL), + }, + A: ip4, + }) + } + } + return rr + + case dns.TypeAAAA: + var rr []dns.RR + for _, ip := range m.IPs { + if ip.To4() != nil { + // TODO(reddaly): IPv4 addresses could be encoded in IPv6 format and + // putinto AAAA records, but the current logic puts ipv4-encodable + // addresses into the A records exclusively. Perhaps this should be + // configurable? + continue + } + + if ip16 := ip.To16(); ip16 != nil { + rr = append(rr, &dns.AAAA{ + Hdr: dns.RR_Header{ + Name: m.HostName, + Rrtype: dns.TypeAAAA, + Class: dns.ClassINET, + Ttl: atomic.LoadUint32(&m.TTL), + }, + AAAA: ip16, + }) + } + } + return rr + + case dns.TypeSRV: + // Create the SRV Record + srv := &dns.SRV{ + Hdr: dns.RR_Header{ + Name: q.Name, + Rrtype: dns.TypeSRV, + Class: dns.ClassINET, + Ttl: atomic.LoadUint32(&m.TTL), + }, + Priority: 10, + Weight: 1, + Port: uint16(m.Port), + Target: m.HostName, + } + recs := []dns.RR{srv} + + // Add the A record + recs = append(recs, m.instanceRecords(dns.Question{ + Name: m.instanceAddr, + Qtype: dns.TypeA, + })...) + + // Add the AAAA record + recs = append(recs, m.instanceRecords(dns.Question{ + Name: m.instanceAddr, + Qtype: dns.TypeAAAA, + })...) + return recs + + case dns.TypeTXT: + txt := &dns.TXT{ + Hdr: dns.RR_Header{ + Name: q.Name, + Rrtype: dns.TypeTXT, + Class: dns.ClassINET, + Ttl: atomic.LoadUint32(&m.TTL), + }, + Txt: m.TXT, + } + return []dns.RR{txt} + } + return nil +} diff --git a/util/mdns/zone_test.go b/util/mdns/zone_test.go new file mode 100644 index 00000000..082d72dd --- /dev/null +++ b/util/mdns/zone_test.go @@ -0,0 +1,275 @@ +package mdns + +import ( + "bytes" + "net" + "reflect" + "testing" + + "github.com/miekg/dns" +) + +func makeService(t *testing.T) *MDNSService { + return makeServiceWithServiceName(t, "_http._tcp") +} + +func makeServiceWithServiceName(t *testing.T, service string) *MDNSService { + m, err := NewMDNSService( + "hostname", + service, + "local.", + "testhost.", + 80, // port + []net.IP{net.IP([]byte{192, 168, 0, 42}), net.ParseIP("2620:0:1000:1900:b0c2:d0b2:c411:18bc")}, + []string{"Local web server"}) // TXT + + if err != nil { + t.Fatalf("err: %v", err) + } + + return m +} + +func TestNewMDNSService_BadParams(t *testing.T) { + for _, test := range []struct { + testName string + hostName string + domain string + }{ + { + "NewMDNSService should fail when passed hostName that is not a legal fully-qualified domain name", + "hostname", // not legal FQDN - should be "hostname." or "hostname.local.", etc. + "local.", // legal + }, + { + "NewMDNSService should fail when passed domain that is not a legal fully-qualified domain name", + "hostname.", // legal + "local", // should be "local." + }, + } { + _, err := NewMDNSService( + "instance name", + "_http._tcp", + test.domain, + test.hostName, + 80, // port + []net.IP{net.IP([]byte{192, 168, 0, 42})}, + []string{"Local web server"}) // TXT + if err == nil { + t.Fatalf("%s: error expected, but got none", test.testName) + } + } +} + +func TestMDNSService_BadAddr(t *testing.T) { + s := makeService(t) + q := dns.Question{ + Name: "random", + Qtype: dns.TypeANY, + } + recs := s.Records(q) + if len(recs) != 0 { + t.Fatalf("bad: %v", recs) + } +} + +func TestMDNSService_ServiceAddr(t *testing.T) { + s := makeService(t) + q := dns.Question{ + Name: "_http._tcp.local.", + Qtype: dns.TypeANY, + } + recs := s.Records(q) + if got, want := len(recs), 5; got != want { + t.Fatalf("got %d records, want %d: %v", got, want, recs) + } + + if ptr, ok := recs[0].(*dns.PTR); !ok { + t.Errorf("recs[0] should be PTR record, got: %v, all records: %v", recs[0], recs) + } else if got, want := ptr.Ptr, "hostname._http._tcp.local."; got != want { + t.Fatalf("bad PTR record %v: got %v, want %v", ptr, got, want) + } + + if _, ok := recs[1].(*dns.SRV); !ok { + t.Errorf("recs[1] should be SRV record, got: %v, all reccords: %v", recs[1], recs) + } + if _, ok := recs[2].(*dns.A); !ok { + t.Errorf("recs[2] should be A record, got: %v, all records: %v", recs[2], recs) + } + if _, ok := recs[3].(*dns.AAAA); !ok { + t.Errorf("recs[3] should be AAAA record, got: %v, all records: %v", recs[3], recs) + } + if _, ok := recs[4].(*dns.TXT); !ok { + t.Errorf("recs[4] should be TXT record, got: %v, all records: %v", recs[4], recs) + } + + q.Qtype = dns.TypePTR + if recs2 := s.Records(q); !reflect.DeepEqual(recs, recs2) { + t.Fatalf("PTR question should return same result as ANY question: ANY => %v, PTR => %v", recs, recs2) + } +} + +func TestMDNSService_InstanceAddr_ANY(t *testing.T) { + s := makeService(t) + q := dns.Question{ + Name: "hostname._http._tcp.local.", + Qtype: dns.TypeANY, + } + recs := s.Records(q) + if len(recs) != 4 { + t.Fatalf("bad: %v", recs) + } + if _, ok := recs[0].(*dns.SRV); !ok { + t.Fatalf("bad: %v", recs[0]) + } + if _, ok := recs[1].(*dns.A); !ok { + t.Fatalf("bad: %v", recs[1]) + } + if _, ok := recs[2].(*dns.AAAA); !ok { + t.Fatalf("bad: %v", recs[2]) + } + if _, ok := recs[3].(*dns.TXT); !ok { + t.Fatalf("bad: %v", recs[3]) + } +} + +func TestMDNSService_InstanceAddr_SRV(t *testing.T) { + s := makeService(t) + q := dns.Question{ + Name: "hostname._http._tcp.local.", + Qtype: dns.TypeSRV, + } + recs := s.Records(q) + if len(recs) != 3 { + t.Fatalf("bad: %v", recs) + } + srv, ok := recs[0].(*dns.SRV) + if !ok { + t.Fatalf("bad: %v", recs[0]) + } + if _, ok := recs[1].(*dns.A); !ok { + t.Fatalf("bad: %v", recs[1]) + } + if _, ok := recs[2].(*dns.AAAA); !ok { + t.Fatalf("bad: %v", recs[2]) + } + + if srv.Port != uint16(s.Port) { + t.Fatalf("bad: %v", recs[0]) + } +} + +func TestMDNSService_InstanceAddr_A(t *testing.T) { + s := makeService(t) + q := dns.Question{ + Name: "hostname._http._tcp.local.", + Qtype: dns.TypeA, + } + recs := s.Records(q) + if len(recs) != 1 { + t.Fatalf("bad: %v", recs) + } + a, ok := recs[0].(*dns.A) + if !ok { + t.Fatalf("bad: %v", recs[0]) + } + if !bytes.Equal(a.A, []byte{192, 168, 0, 42}) { + t.Fatalf("bad: %v", recs[0]) + } +} + +func TestMDNSService_InstanceAddr_AAAA(t *testing.T) { + s := makeService(t) + q := dns.Question{ + Name: "hostname._http._tcp.local.", + Qtype: dns.TypeAAAA, + } + recs := s.Records(q) + if len(recs) != 1 { + t.Fatalf("bad: %v", recs) + } + a4, ok := recs[0].(*dns.AAAA) + if !ok { + t.Fatalf("bad: %v", recs[0]) + } + ip6 := net.ParseIP("2620:0:1000:1900:b0c2:d0b2:c411:18bc") + if got := len(ip6); got != net.IPv6len { + t.Fatalf("test IP failed to parse (len = %d, want %d)", got, net.IPv6len) + } + if !bytes.Equal(a4.AAAA, ip6) { + t.Fatalf("bad: %v", recs[0]) + } +} + +func TestMDNSService_InstanceAddr_TXT(t *testing.T) { + s := makeService(t) + q := dns.Question{ + Name: "hostname._http._tcp.local.", + Qtype: dns.TypeTXT, + } + recs := s.Records(q) + if len(recs) != 1 { + t.Fatalf("bad: %v", recs) + } + txt, ok := recs[0].(*dns.TXT) + if !ok { + t.Fatalf("bad: %v", recs[0]) + } + if got, want := txt.Txt, s.TXT; !reflect.DeepEqual(got, want) { + t.Fatalf("TXT record mismatch for %v: got %v, want %v", recs[0], got, want) + } +} + +func TestMDNSService_HostNameQuery(t *testing.T) { + s := makeService(t) + for _, test := range []struct { + q dns.Question + want []dns.RR + }{ + { + dns.Question{Name: "testhost.", Qtype: dns.TypeA}, + []dns.RR{&dns.A{ + Hdr: dns.RR_Header{ + Name: "testhost.", + Rrtype: dns.TypeA, + Class: dns.ClassINET, + Ttl: 120, + }, + A: net.IP([]byte{192, 168, 0, 42}), + }}, + }, + { + dns.Question{Name: "testhost.", Qtype: dns.TypeAAAA}, + []dns.RR{&dns.AAAA{ + Hdr: dns.RR_Header{ + Name: "testhost.", + Rrtype: dns.TypeAAAA, + Class: dns.ClassINET, + Ttl: 120, + }, + AAAA: net.ParseIP("2620:0:1000:1900:b0c2:d0b2:c411:18bc"), + }}, + }, + } { + if got := s.Records(test.q); !reflect.DeepEqual(got, test.want) { + t.Errorf("hostname query failed: s.Records(%v) = %v, want %v", test.q, got, test.want) + } + } +} + +func TestMDNSService_serviceEnum_PTR(t *testing.T) { + s := makeService(t) + q := dns.Question{ + Name: "_services._dns-sd._udp.local.", + Qtype: dns.TypePTR, + } + recs := s.Records(q) + if len(recs) != 1 { + t.Fatalf("bad: %v", recs) + } + if ptr, ok := recs[0].(*dns.PTR); !ok { + t.Errorf("recs[0] should be PTR record, got: %v, all records: %v", recs[0], recs) + } else if got, want := ptr.Ptr, "_http._tcp.local."; got != want { + t.Fatalf("bad PTR record %v: got %v, want %v", ptr, got, want) + } +} From ec80ceb8c2ba3d447c1c638090614bc19626fc99 Mon Sep 17 00:00:00 2001 From: Asim Aslam Date: Sat, 11 Apr 2020 18:23:37 +0100 Subject: [PATCH 30/44] Update readme --- README.md | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.md b/README.md index d6294ace..d6b39afc 100644 --- a/README.md +++ b/README.md @@ -35,8 +35,7 @@ communication. A request made to a service will be automatically resolved, load transport is [gRPC](https://grpc.io/). - **Async Messaging** - PubSub is built in as a first class citizen for asynchronous communication and event driven architectures. -Event notifications are a core pattern in micro service development. The default messaging system is an embedded [NATS](https://nats.io/) -server. +Event notifications are a core pattern in micro service development. The default messaging system is a HTTP event message broker. - **Pluggable Interfaces** - Go Micro makes use of Go interfaces for each distributed system abstraction. Because of this these interfaces are pluggable and allows Go Micro to be runtime agnostic. You can plugin any underlying technology. Find plugins in From 0a2363b49ba351fb0093ab3c7205bb216b5d1c73 Mon Sep 17 00:00:00 2001 From: Vasiliy Tolstov Date: Sat, 11 Apr 2020 22:21:55 +0300 Subject: [PATCH 31/44] api minor improvements (#1526) * api/handler/rpc: unblock all http methods and set Host meta * api/router/static: add debug log Signed-off-by: Vasiliy Tolstov --- api/handler/rpc/rpc.go | 6 +++--- api/router/static/static.go | 6 ++++-- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/api/handler/rpc/rpc.go b/api/handler/rpc/rpc.go index f4d23bc9..980333b5 100644 --- a/api/handler/rpc/rpc.go +++ b/api/handler/rpc/rpc.go @@ -100,8 +100,7 @@ func (h *rpcHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { return } - // only allow post when we have the router - if r.Method != "GET" && (h.opts.Router != nil && r.Method != "POST") { + if h.opts.Router == nil && r.Method != "GET" { writeError(w, r, errors.MethodNotAllowed("go.micro.api", "method not allowed")) return } @@ -123,7 +122,8 @@ func (h *rpcHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { if !ok { md = make(metadata.Metadata) } - + // fill contex with http headers + md["Host"] = r.Host // merge context with overwrite cx = metadata.MergeContext(cx, md, true) diff --git a/api/router/static/static.go b/api/router/static/static.go index f868e52d..0c659c31 100644 --- a/api/router/static/static.go +++ b/api/router/static/static.go @@ -264,7 +264,9 @@ func (r *staticRouter) endpoint(req *http.Request) (*endpoint, error) { for _, pathreg := range ep.pathregs { matches, err := pathreg.Match(path, "") if err != nil { - // TODO: log error + if logger.V(logger.DebugLevel, logger.DefaultLogger) { + logger.Debugf("api path not match %s != %v", path, pathreg) + } continue } pMatch = true @@ -290,7 +292,7 @@ func (r *staticRouter) endpoint(req *http.Request) (*endpoint, error) { } // no match - return nil, fmt.Errorf("endpoint not found for %v", req) + return nil, fmt.Errorf("endpoint not found for %v", req.URL) } func (r *staticRouter) Route(req *http.Request) (*api.Service, error) { From 3ce2ab88f553bf490a8af54da670315c014cd130 Mon Sep 17 00:00:00 2001 From: Vasiliy Tolstov Date: Sat, 11 Apr 2020 22:37:29 +0300 Subject: [PATCH 32/44] broker/nats: remove embed nats server reference (#1527) Signed-off-by: Vasiliy Tolstov --- broker/nats/nats.go | 222 ++++------------------------------------- broker/nats/options.go | 6 -- go.mod | 2 +- 3 files changed, 19 insertions(+), 211 deletions(-) diff --git a/broker/nats/nats.go b/broker/nats/nats.go index 01b59d18..b8bf6daf 100644 --- a/broker/nats/nats.go +++ b/broker/nats/nats.go @@ -4,19 +4,13 @@ package nats import ( "context" "errors" - "net" - "net/url" - "strconv" "strings" "sync" - "time" "github.com/micro/go-micro/v2/broker" "github.com/micro/go-micro/v2/codec/json" "github.com/micro/go-micro/v2/logger" "github.com/micro/go-micro/v2/registry" - "github.com/micro/go-micro/v2/util/addr" - "github.com/nats-io/nats-server/v2/server" nats "github.com/nats-io/nats.go" ) @@ -35,13 +29,6 @@ type natsBroker struct { // should we drain the connection drain bool closeCh chan (error) - - // embedded server - server *server.Server - // configure to use local server - local bool - // server exit channel - exit chan bool } type subscriber struct { @@ -108,186 +95,18 @@ func (n *natsBroker) setAddrs(addrs []string) []string { } cAddrs = append(cAddrs, addr) } - // if there's no address and we weren't told to - // embed a local server then use the default url - if len(cAddrs) == 0 && !n.local { + if len(cAddrs) == 0 { cAddrs = []string{nats.DefaultURL} } return cAddrs } -// serve stats a local nats server if needed -func (n *natsBroker) serve(exit chan bool) error { - var host string - var port int - var local bool - - // with no address we just default it - // this is a local client address - if len(n.addrs) == 0 { - // find an advertiseable ip - if h, err := addr.Extract(""); err != nil { - host = "127.0.0.1" - } else { - host = h - } - - port = -1 - local = true - } else { - address := n.addrs[0] - if strings.HasPrefix(address, "nats://") { - address = strings.TrimPrefix(address, "nats://") - } - - // check if its a local address and only then embed - if addr.IsLocal(address) { - h, p, err := net.SplitHostPort(address) - if err == nil { - host = h - port, _ = strconv.Atoi(p) - local = true - } - } - } - - // we only setup a server for local things - if !local { - return nil - } - - // 1. create new server - // 2. register the server - // 3. connect to other servers - var cOpts server.ClusterOpts - var routes []*url.URL - - // get existing nats servers to connect to - services, err := n.opts.Registry.GetService("go.micro.nats.broker") - if err == nil { - for _, service := range services { - for _, node := range service.Nodes { - u, err := url.Parse("nats://" + node.Address) - if err != nil { - if logger.V(logger.ErrorLevel, logger.DefaultLogger) { - logger.Error(err) - } - continue - } - // append to the cluster routes - routes = append(routes, u) - } - } - } - - // try get existing server - s := n.server - - // get a host address - caddr, err := addr.Extract("") - if err != nil { - caddr = "0.0.0.0" - } - - // set cluster opts - cOpts = server.ClusterOpts{ - Host: caddr, - Port: -1, - } - - if s == nil { - var err error - s, err = server.NewServer(&server.Options{ - // Specify the host - Host: host, - // Use a random port - Port: port, - // Set the cluster ops - Cluster: cOpts, - // Set the routes - Routes: routes, - NoLog: true, - NoSigs: true, - MaxControlLine: 2048, - TLSConfig: n.opts.TLSConfig, - }) - if err != nil { - return err - } - - // save the server - n.server = s - } - - // start the server - go s.Start() - - var ready bool - - // wait till its ready for connections - for i := 0; i < 3; i++ { - if s.ReadyForConnections(time.Second) { - ready = true - break - } - } - - if !ready { - return errors.New("server not ready") - } - - // set the client address - n.addrs = []string{s.ClientURL()} - - go func() { - // register the cluster address - for { - select { - case <-exit: - // deregister on exit - n.opts.Registry.Deregister(®istry.Service{ - Name: "go.micro.nats.broker", - Version: "v2", - Nodes: []*registry.Node{ - {Id: s.ID(), Address: s.ClusterAddr().String()}, - }, - }) - s.Shutdown() - return - default: - // register the broker - n.opts.Registry.Register(®istry.Service{ - Name: "go.micro.nats.broker", - Version: "v2", - Nodes: []*registry.Node{ - {Id: s.ID(), Address: s.ClusterAddr().String()}, - }, - }, registry.RegisterTTL(time.Minute)) - time.Sleep(time.Minute) - } - } - }() - - return nil -} - func (n *natsBroker) Connect() error { n.Lock() defer n.Unlock() - if !n.connected { - // create exit chan - n.exit = make(chan bool) - - // start embedded server if asked to - if n.local { - if err := n.serve(n.exit); err != nil { - return err - } - } - - // set to connected - n.connected = true + if n.connected { + return nil } status := nats.CLOSED @@ -297,6 +116,7 @@ func (n *natsBroker) Connect() error { switch status { case nats.CONNECTED, nats.RECONNECTING, nats.CONNECTING: + n.connected = true return nil default: // DISCONNECTED or CLOSED or DRAINING opts := n.nopts @@ -314,13 +134,14 @@ func (n *natsBroker) Connect() error { return err } n.conn = c + n.connected = true return nil } } func (n *natsBroker) Disconnect() error { - n.RLock() - defer n.RUnlock() + n.Lock() + defer n.Unlock() // drain the connection if specified if n.drain { @@ -331,16 +152,6 @@ func (n *natsBroker) Disconnect() error { // close the client connection n.conn.Close() - // shutdown the local server - // and deregister - if n.server != nil { - select { - case <-n.exit: - default: - close(n.exit) - } - } - // set not connected n.connected = false @@ -357,19 +168,27 @@ func (n *natsBroker) Options() broker.Options { } func (n *natsBroker) Publish(topic string, msg *broker.Message, opts ...broker.PublishOption) error { + n.RLock() + defer n.RUnlock() + + if n.conn == nil { + return errors.New("not connected") + } + b, err := n.opts.Codec.Marshal(msg) if err != nil { return err } - n.RLock() - defer n.RUnlock() return n.conn.Publish(topic, b) } func (n *natsBroker) Subscribe(topic string, handler broker.Handler, opts ...broker.SubscribeOption) (broker.Subscriber, error) { + n.RLock() if n.conn == nil { + n.RUnlock() return nil, errors.New("not connected") } + n.RUnlock() opt := broker.SubscribeOptions{ AutoAck: true, @@ -441,15 +260,10 @@ func (n *natsBroker) setOption(opts ...broker.Option) { n.nopts = nopts } - local, ok := n.opts.Context.Value(localServerKey{}).(bool) - if ok { - n.local = local - } - // broker.Options have higher priority than nats.Options // only if Addrs, Secure or TLSConfig were not set through a broker.Option // we read them from nats.Option - if len(n.opts.Addrs) == 0 && !n.local { + if len(n.opts.Addrs) == 0 { n.opts.Addrs = n.nopts.Servers } diff --git a/broker/nats/options.go b/broker/nats/options.go index ba127e73..78708c0b 100644 --- a/broker/nats/options.go +++ b/broker/nats/options.go @@ -7,18 +7,12 @@ import ( type optionsKey struct{} type drainConnectionKey struct{} -type localServerKey struct{} // Options accepts nats.Options func Options(opts nats.Options) broker.Option { return setBrokerOption(optionsKey{}, opts) } -// LocalServer embeds a local server rather than connecting to one -func LocalServer() broker.Option { - return setBrokerOption(localServerKey{}, true) -} - // DrainConnection will drain subscription on close func DrainConnection() broker.Option { return setBrokerOption(drainConnectionKey{}, struct{}{}) diff --git a/go.mod b/go.mod index 8a445a11..ffed3e84 100644 --- a/go.mod +++ b/go.mod @@ -48,7 +48,7 @@ require ( github.com/micro/cli/v2 v2.1.2 github.com/miekg/dns v1.1.27 github.com/mitchellh/hashstructure v1.0.0 - github.com/nats-io/nats-server/v2 v2.1.6 + github.com/nats-io/nats-server/v2 v2.1.6 // indirect github.com/nats-io/nats.go v1.9.2 github.com/nlopes/slack v0.6.1-0.20191106133607-d06c2a2b3249 github.com/oxtoacart/bpool v0.0.0-20190530202638-03653db5a59c From 4e539361fa704bad168305e92b7d5c0a17ab70c9 Mon Sep 17 00:00:00 2001 From: Asim Aslam Date: Sun, 12 Apr 2020 10:58:12 +0100 Subject: [PATCH 33/44] strip file --- registry/registry.go | 27 +++++++++++++++++++++++++++ registry/service.go | 28 ---------------------------- 2 files changed, 27 insertions(+), 28 deletions(-) delete mode 100644 registry/service.go diff --git a/registry/registry.go b/registry/registry.go index 6ec72292..291a1988 100644 --- a/registry/registry.go +++ b/registry/registry.go @@ -28,6 +28,33 @@ type Registry interface { String() string } +type Service struct { + Name string `json:"name"` + Version string `json:"version"` + Metadata map[string]string `json:"metadata"` + Endpoints []*Endpoint `json:"endpoints"` + Nodes []*Node `json:"nodes"` +} + +type Node struct { + Id string `json:"id"` + Address string `json:"address"` + Metadata map[string]string `json:"metadata"` +} + +type Endpoint struct { + Name string `json:"name"` + Request *Value `json:"request"` + Response *Value `json:"response"` + Metadata map[string]string `json:"metadata"` +} + +type Value struct { + Name string `json:"name"` + Type string `json:"type"` + Values []*Value `json:"values"` +} + type Option func(*Options) type RegisterOption func(*RegisterOptions) diff --git a/registry/service.go b/registry/service.go deleted file mode 100644 index 5259a6b6..00000000 --- a/registry/service.go +++ /dev/null @@ -1,28 +0,0 @@ -package registry - -type Service struct { - Name string `json:"name"` - Version string `json:"version"` - Metadata map[string]string `json:"metadata"` - Endpoints []*Endpoint `json:"endpoints"` - Nodes []*Node `json:"nodes"` -} - -type Node struct { - Id string `json:"id"` - Address string `json:"address"` - Metadata map[string]string `json:"metadata"` -} - -type Endpoint struct { - Name string `json:"name"` - Request *Value `json:"request"` - Response *Value `json:"response"` - Metadata map[string]string `json:"metadata"` -} - -type Value struct { - Name string `json:"name"` - Type string `json:"type"` - Values []*Value `json:"values"` -} From cf67d460b761109e22d9e5755b1d6d77d5ad305e Mon Sep 17 00:00:00 2001 From: Asim Aslam Date: Sun, 12 Apr 2020 11:01:09 +0100 Subject: [PATCH 34/44] strip down mdns watcher --- registry/mdns_registry.go | 79 ++++++++++++++++++ registry/mdns_test.go | 142 ++++++++++++++++++++++++++++++++ registry/mdns_watcher.go | 87 -------------------- registry/mdns_watcher_test.go | 149 ---------------------------------- 4 files changed, 221 insertions(+), 236 deletions(-) delete mode 100644 registry/mdns_watcher.go delete mode 100644 registry/mdns_watcher_test.go diff --git a/registry/mdns_registry.go b/registry/mdns_registry.go index dfc8e6f0..18758878 100644 --- a/registry/mdns_registry.go +++ b/registry/mdns_registry.go @@ -54,6 +54,17 @@ type mdnsRegistry struct { listener chan *mdns.ServiceEntry } +type mdnsWatcher struct { + id string + wo WatchOptions + ch chan *mdns.ServiceEntry + exit chan struct{} + // the mdns domain + domain string + // the registry + registry *mdnsRegistry +} + func encode(txt *mdnsTxt) ([]string, error) { b, err := json.Marshal(txt) if err != nil { @@ -534,6 +545,74 @@ func (m *mdnsRegistry) String() string { return "mdns" } +func (m *mdnsWatcher) Next() (*Result, error) { + for { + select { + case e := <-m.ch: + txt, err := decode(e.InfoFields) + if err != nil { + continue + } + + if len(txt.Service) == 0 || len(txt.Version) == 0 { + continue + } + + // Filter watch options + // wo.Service: Only keep services we care about + if len(m.wo.Service) > 0 && txt.Service != m.wo.Service { + continue + } + + var action string + + if e.TTL == 0 { + action = "delete" + } else { + action = "create" + } + + service := &Service{ + Name: txt.Service, + Version: txt.Version, + Endpoints: txt.Endpoints, + } + + // skip anything without the domain we care about + suffix := fmt.Sprintf(".%s.%s.", service.Name, m.domain) + if !strings.HasSuffix(e.Name, suffix) { + continue + } + + service.Nodes = append(service.Nodes, &Node{ + Id: strings.TrimSuffix(e.Name, suffix), + Address: fmt.Sprintf("%s:%d", e.AddrV4.String(), e.Port), + Metadata: txt.Metadata, + }) + + return &Result{ + Action: action, + Service: service, + }, nil + case <-m.exit: + return nil, ErrWatcherStopped + } + } +} + +func (m *mdnsWatcher) Stop() { + select { + case <-m.exit: + return + default: + close(m.exit) + // remove self from the registry + m.registry.mtx.Lock() + delete(m.registry.watchers, m.id) + m.registry.mtx.Unlock() + } +} + // NewRegistry returns a new default registry which is mdns func NewRegistry(opts ...Option) Registry { return newRegistry(opts...) diff --git a/registry/mdns_test.go b/registry/mdns_test.go index c9979d5c..5f5cc617 100644 --- a/registry/mdns_test.go +++ b/registry/mdns_test.go @@ -197,3 +197,145 @@ func TestEncoding(t *testing.T) { } } + +func TestWatcher(t *testing.T) { + if travis := os.Getenv("TRAVIS"); travis == "true" { + t.Skip() + } + + testData := []*Service{ + { + Name: "test1", + Version: "1.0.1", + Nodes: []*Node{ + { + Id: "test1-1", + Address: "10.0.0.1:10001", + Metadata: map[string]string{ + "foo": "bar", + }, + }, + }, + }, + { + Name: "test2", + Version: "1.0.2", + Nodes: []*Node{ + { + Id: "test2-1", + Address: "10.0.0.2:10002", + Metadata: map[string]string{ + "foo2": "bar2", + }, + }, + }, + }, + { + Name: "test3", + Version: "1.0.3", + Nodes: []*Node{ + { + Id: "test3-1", + Address: "10.0.0.3:10003", + Metadata: map[string]string{ + "foo3": "bar3", + }, + }, + }, + }, + } + + testFn := func(service, s *Service) { + if s == nil { + t.Fatalf("Expected one result for %s got nil", service.Name) + + } + + if s.Name != service.Name { + t.Fatalf("Expected name %s got %s", service.Name, s.Name) + } + + if s.Version != service.Version { + t.Fatalf("Expected version %s got %s", service.Version, s.Version) + } + + if len(s.Nodes) != 1 { + t.Fatalf("Expected 1 node, got %d", len(s.Nodes)) + } + + node := s.Nodes[0] + + if node.Id != service.Nodes[0].Id { + t.Fatalf("Expected node id %s got %s", service.Nodes[0].Id, node.Id) + } + + if node.Address != service.Nodes[0].Address { + t.Fatalf("Expected node address %s got %s", service.Nodes[0].Address, node.Address) + } + } + + travis := os.Getenv("TRAVIS") + + var opts []Option + + if travis == "true" { + opts = append(opts, Timeout(time.Millisecond*100)) + } + + // new registry + r := NewRegistry(opts...) + + w, err := r.Watch() + if err != nil { + t.Fatal(err) + } + defer w.Stop() + + for _, service := range testData { + // register service + if err := r.Register(service); err != nil { + t.Fatal(err) + } + + for { + res, err := w.Next() + if err != nil { + t.Fatal(err) + } + + if res.Service.Name != service.Name { + continue + } + + if res.Action != "create" { + t.Fatalf("Expected create event got %s for %s", res.Action, res.Service.Name) + } + + testFn(service, res.Service) + break + } + + // deregister + if err := r.Deregister(service); err != nil { + t.Fatal(err) + } + + for { + res, err := w.Next() + if err != nil { + t.Fatal(err) + } + + if res.Service.Name != service.Name { + continue + } + + if res.Action != "delete" { + continue + } + + testFn(service, res.Service) + break + } + } +} diff --git a/registry/mdns_watcher.go b/registry/mdns_watcher.go deleted file mode 100644 index e0ef4a48..00000000 --- a/registry/mdns_watcher.go +++ /dev/null @@ -1,87 +0,0 @@ -package registry - -import ( - "fmt" - "strings" - - "github.com/micro/go-micro/v2/util/mdns" -) - -type mdnsWatcher struct { - id string - wo WatchOptions - ch chan *mdns.ServiceEntry - exit chan struct{} - // the mdns domain - domain string - // the registry - registry *mdnsRegistry -} - -func (m *mdnsWatcher) Next() (*Result, error) { - for { - select { - case e := <-m.ch: - txt, err := decode(e.InfoFields) - if err != nil { - continue - } - - if len(txt.Service) == 0 || len(txt.Version) == 0 { - continue - } - - // Filter watch options - // wo.Service: Only keep services we care about - if len(m.wo.Service) > 0 && txt.Service != m.wo.Service { - continue - } - - var action string - - if e.TTL == 0 { - action = "delete" - } else { - action = "create" - } - - service := &Service{ - Name: txt.Service, - Version: txt.Version, - Endpoints: txt.Endpoints, - } - - // skip anything without the domain we care about - suffix := fmt.Sprintf(".%s.%s.", service.Name, m.domain) - if !strings.HasSuffix(e.Name, suffix) { - continue - } - - service.Nodes = append(service.Nodes, &Node{ - Id: strings.TrimSuffix(e.Name, suffix), - Address: fmt.Sprintf("%s:%d", e.AddrV4.String(), e.Port), - Metadata: txt.Metadata, - }) - - return &Result{ - Action: action, - Service: service, - }, nil - case <-m.exit: - return nil, ErrWatcherStopped - } - } -} - -func (m *mdnsWatcher) Stop() { - select { - case <-m.exit: - return - default: - close(m.exit) - // remove self from the registry - m.registry.mtx.Lock() - delete(m.registry.watchers, m.id) - m.registry.mtx.Unlock() - } -} diff --git a/registry/mdns_watcher_test.go b/registry/mdns_watcher_test.go deleted file mode 100644 index bd837c58..00000000 --- a/registry/mdns_watcher_test.go +++ /dev/null @@ -1,149 +0,0 @@ -package registry - -import ( - "os" - "testing" - "time" -) - -func TestWatcher(t *testing.T) { - if travis := os.Getenv("TRAVIS"); travis == "true" { - t.Skip() - } - - testData := []*Service{ - { - Name: "test1", - Version: "1.0.1", - Nodes: []*Node{ - { - Id: "test1-1", - Address: "10.0.0.1:10001", - Metadata: map[string]string{ - "foo": "bar", - }, - }, - }, - }, - { - Name: "test2", - Version: "1.0.2", - Nodes: []*Node{ - { - Id: "test2-1", - Address: "10.0.0.2:10002", - Metadata: map[string]string{ - "foo2": "bar2", - }, - }, - }, - }, - { - Name: "test3", - Version: "1.0.3", - Nodes: []*Node{ - { - Id: "test3-1", - Address: "10.0.0.3:10003", - Metadata: map[string]string{ - "foo3": "bar3", - }, - }, - }, - }, - } - - testFn := func(service, s *Service) { - if s == nil { - t.Fatalf("Expected one result for %s got nil", service.Name) - - } - - if s.Name != service.Name { - t.Fatalf("Expected name %s got %s", service.Name, s.Name) - } - - if s.Version != service.Version { - t.Fatalf("Expected version %s got %s", service.Version, s.Version) - } - - if len(s.Nodes) != 1 { - t.Fatalf("Expected 1 node, got %d", len(s.Nodes)) - } - - node := s.Nodes[0] - - if node.Id != service.Nodes[0].Id { - t.Fatalf("Expected node id %s got %s", service.Nodes[0].Id, node.Id) - } - - if node.Address != service.Nodes[0].Address { - t.Fatalf("Expected node address %s got %s", service.Nodes[0].Address, node.Address) - } - } - - travis := os.Getenv("TRAVIS") - - var opts []Option - - if travis == "true" { - opts = append(opts, Timeout(time.Millisecond*100)) - } - - // new registry - r := NewRegistry(opts...) - - w, err := r.Watch() - if err != nil { - t.Fatal(err) - } - defer w.Stop() - - for _, service := range testData { - // register service - if err := r.Register(service); err != nil { - t.Fatal(err) - } - - for { - res, err := w.Next() - if err != nil { - t.Fatal(err) - } - - if res.Service.Name != service.Name { - continue - } - - if res.Action != "create" { - t.Fatalf("Expected create event got %s for %s", res.Action, res.Service.Name) - } - - testFn(service, res.Service) - break - } - - // deregister - if err := r.Deregister(service); err != nil { - t.Fatal(err) - } - - for { - res, err := w.Next() - if err != nil { - t.Fatal(err) - } - - if res.Service.Name != service.Name { - continue - } - - if res.Action != "delete" { - continue - } - - testFn(service, res.Service) - break - } - } -} From 962588b64970f11c98b79d3cd6345821a97a0ec5 Mon Sep 17 00:00:00 2001 From: Asim Aslam Date: Sun, 12 Apr 2020 11:16:08 +0100 Subject: [PATCH 35/44] Strip MetadataKey global var --- api/handler/rpc/rpc.go | 4 ++-- api/router/static/static.go | 4 ++-- metadata/metadata.go | 12 ++++++------ sync/sync.go | 10 +++++----- 4 files changed, 15 insertions(+), 15 deletions(-) diff --git a/api/handler/rpc/rpc.go b/api/handler/rpc/rpc.go index 980333b5..7c06b663 100644 --- a/api/handler/rpc/rpc.go +++ b/api/handler/rpc/rpc.go @@ -118,7 +118,7 @@ func (h *rpcHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { // create context cx := ctx.FromRequest(r) // get context from http handler wrappers - md, ok := r.Context().Value(metadata.MetadataKey{}).(metadata.Metadata) + md, ok := metadata.FromContext(r.Context()) if !ok { md = make(metadata.Metadata) } @@ -293,7 +293,7 @@ func requestPayload(r *http.Request) ([]byte, error) { // otherwise as per usual ctx := r.Context() // dont user meadata.FromContext as it mangles names - md, ok := ctx.Value(metadata.MetadataKey{}).(metadata.Metadata) + md, ok := metadata.FromContext(ctx) if !ok { md = make(map[string]string) } diff --git a/api/router/static/static.go b/api/router/static/static.go index 0c659c31..2b0218f1 100644 --- a/api/router/static/static.go +++ b/api/router/static/static.go @@ -271,7 +271,7 @@ func (r *staticRouter) endpoint(req *http.Request) (*endpoint, error) { } pMatch = true ctx := req.Context() - md, ok := ctx.Value(metadata.MetadataKey{}).(metadata.Metadata) + md, ok := metadata.FromContext(ctx) if !ok { md = make(metadata.Metadata) } @@ -279,7 +279,7 @@ func (r *staticRouter) endpoint(req *http.Request) (*endpoint, error) { md[fmt.Sprintf("x-api-field-%s", k)] = v } md["x-api-body"] = ep.apiep.Body - *req = *req.Clone(context.WithValue(ctx, metadata.MetadataKey{}, md)) + *req = *req.Clone(metadata.NewContext(ctx, md)) break pathLoop } if !pMatch { diff --git a/metadata/metadata.go b/metadata/metadata.go index bd539314..c99766a8 100644 --- a/metadata/metadata.go +++ b/metadata/metadata.go @@ -6,7 +6,7 @@ import ( "strings" ) -type MetadataKey struct{} +type metadataKey struct{} // Metadata is our way of representing request headers internally. // They're used at the RPC level and translate back and forth @@ -57,7 +57,7 @@ func Set(ctx context.Context, k, v string) context.Context { } else { md[k] = v } - return context.WithValue(ctx, MetadataKey{}, md) + return context.WithValue(ctx, metadataKey{}, md) } // Get returns a single value from metadata in the context @@ -80,7 +80,7 @@ func Get(ctx context.Context, key string) (string, bool) { // FromContext returns metadata from the given context func FromContext(ctx context.Context) (Metadata, bool) { - md, ok := ctx.Value(MetadataKey{}).(Metadata) + md, ok := ctx.Value(metadataKey{}).(Metadata) if !ok { return nil, ok } @@ -96,7 +96,7 @@ func FromContext(ctx context.Context) (Metadata, bool) { // NewContext creates a new context with the given metadata func NewContext(ctx context.Context, md Metadata) context.Context { - return context.WithValue(ctx, MetadataKey{}, md) + return context.WithValue(ctx, metadataKey{}, md) } // MergeContext merges metadata to existing metadata, overwriting if specified @@ -104,7 +104,7 @@ func MergeContext(ctx context.Context, patchMd Metadata, overwrite bool) context if ctx == nil { ctx = context.Background() } - md, _ := ctx.Value(MetadataKey{}).(Metadata) + md, _ := ctx.Value(metadataKey{}).(Metadata) cmd := make(Metadata, len(md)) for k, v := range md { cmd[k] = v @@ -118,5 +118,5 @@ func MergeContext(ctx context.Context, patchMd Metadata, overwrite bool) context delete(cmd, k) } } - return context.WithValue(ctx, MetadataKey{}, cmd) + return context.WithValue(ctx, metadataKey{}, cmd) } diff --git a/sync/sync.go b/sync/sync.go index ebaec6da..0c5203b7 100644 --- a/sync/sync.go +++ b/sync/sync.go @@ -28,10 +28,10 @@ type Sync interface { // Leader provides leadership election type Leader interface { - // resign leadership - Resign() error - // status returns when leadership is lost - Status() chan bool + // resign leadership + Resign() error + // status returns when leadership is lost + Status() chan bool } type Options struct { @@ -41,7 +41,7 @@ type Options struct { type Option func(o *Options) -type LeaderOptions struct {} +type LeaderOptions struct{} type LeaderOption func(o *LeaderOptions) From 08ca61c12193f807feced9b0128640fa95b519c2 Mon Sep 17 00:00:00 2001 From: Asim Aslam Date: Sun, 12 Apr 2020 11:17:23 +0100 Subject: [PATCH 36/44] add metadata set --- metadata/metadata.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/metadata/metadata.go b/metadata/metadata.go index c99766a8..6fcb8199 100644 --- a/metadata/metadata.go +++ b/metadata/metadata.go @@ -25,6 +25,10 @@ func (md Metadata) Get(key string) (string, bool) { return val, ok } +func (md Metadata) Set(key, val string) { + md[key] = val +} + func (md Metadata) Delete(key string) { // delete key as-is delete(md, key) From d03a02f2e41f4d9826fa568e5ee96dd7958a4cc6 Mon Sep 17 00:00:00 2001 From: Asim Aslam Date: Sun, 12 Apr 2020 11:25:12 +0100 Subject: [PATCH 37/44] fix import --- api/router/static/static.go | 1 - 1 file changed, 1 deletion(-) diff --git a/api/router/static/static.go b/api/router/static/static.go index 2b0218f1..fd630cc2 100644 --- a/api/router/static/static.go +++ b/api/router/static/static.go @@ -1,7 +1,6 @@ package static import ( - "context" "errors" "fmt" "net/http" From b08c636b44e22c56882fed8f705b26ce5ae6f0f8 Mon Sep 17 00:00:00 2001 From: Asim Aslam Date: Sun, 12 Apr 2020 14:29:38 +0100 Subject: [PATCH 38/44] fixup handler tests --- api/grpc_test.go | 60 ++++++++++++-------------------------- api/handler/api/api.go | 2 +- api/handler/event/event.go | 2 +- api/handler/options.go | 15 +++++----- api/handler/rpc/rpc.go | 2 +- 5 files changed, 28 insertions(+), 53 deletions(-) diff --git a/api/grpc_test.go b/api/grpc_test.go index f40734c3..e31f3759 100644 --- a/api/grpc_test.go +++ b/api/grpc_test.go @@ -9,20 +9,16 @@ import ( "testing" "time" - "github.com/micro/go-micro/v2" "github.com/micro/go-micro/v2/api" - ahandler "github.com/micro/go-micro/v2/api/handler" - apirpc "github.com/micro/go-micro/v2/api/handler/rpc" + "github.com/micro/go-micro/v2/api/handler" + "github.com/micro/go-micro/v2/api/handler/rpc" "github.com/micro/go-micro/v2/api/router" rstatic "github.com/micro/go-micro/v2/api/router/static" - bmemory "github.com/micro/go-micro/v2/broker/memory" "github.com/micro/go-micro/v2/client" gcli "github.com/micro/go-micro/v2/client/grpc" rmemory "github.com/micro/go-micro/v2/registry/memory" "github.com/micro/go-micro/v2/server" gsrv "github.com/micro/go-micro/v2/server/grpc" - tgrpc "github.com/micro/go-micro/v2/transport/grpc" - pb "github.com/micro/go-micro/v2/server/grpc/proto" ) @@ -39,49 +35,33 @@ func (s *testServer) Call(ctx context.Context, req *pb.Request, rsp *pb.Response func TestApiAndGRPC(t *testing.T) { r := rmemory.NewRegistry() - b := bmemory.NewBroker() - tr := tgrpc.NewTransport() + + // create a new client s := gsrv.NewServer( - server.Broker(b), server.Name("foo"), server.Registry(r), - server.Transport(tr), ) + + // create a new server c := gcli.NewClient( client.Registry(r), - client.Broker(b), - client.Transport(tr), ) - ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) - defer cancel() - svc := micro.NewService( - micro.Server(s), - micro.Client(c), - micro.Broker(b), - micro.Registry(r), - micro.Transport(tr), - micro.Context(ctx)) h := &testServer{} pb.RegisterTestHandler(s, h) - go func() { - if err := svc.Run(); err != nil { - t.Fatalf("failed to start: %v", err) - } - }() - time.Sleep(1 * time.Second) - // check registration - services, err := r.GetService("foo") - if err != nil || len(services) == 0 { - t.Fatalf("failed to get service: %v # %d", err, len(services)) + if err := s.Start(); err != nil { + t.Fatalf("failed to start: %v", err) } + defer s.Stop() + // create a new router router := rstatic.NewRouter( - router.WithHandler(apirpc.Handler), - router.WithRegistry(svc.Server().Options().Registry), + router.WithHandler(rpc.Handler), + router.WithRegistry(r), ) - err = router.Register(&api.Endpoint{ + + err := router.Register(&api.Endpoint{ Name: "foo.Test.Call", Method: []string{"GET"}, Path: []string{"/api/v0/test/call/{name}"}, @@ -91,9 +71,9 @@ func TestApiAndGRPC(t *testing.T) { t.Fatal(err) } - hrpc := apirpc.NewHandler( - ahandler.WithService(svc), - ahandler.WithRouter(router), + hrpc := rpc.NewHandler( + handler.WithClient(c), + handler.WithRouter(router), ) hsrv := &http.Server{ @@ -115,6 +95,7 @@ func TestApiAndGRPC(t *testing.T) { t.Fatalf("Failed to created http.Request: %v", err) } defer rsp.Body.Close() + buf, err := ioutil.ReadAll(rsp.Body) if err != nil { t.Fatal(err) @@ -124,9 +105,4 @@ func TestApiAndGRPC(t *testing.T) { if string(buf) != jsonMsg { t.Fatalf("invalid message received, parsing error %s != %s", buf, jsonMsg) } - select { - case <-ctx.Done(): - return - } - } diff --git a/api/handler/api/api.go b/api/handler/api/api.go index 85c7be75..53d707ed 100644 --- a/api/handler/api/api.go +++ b/api/handler/api/api.go @@ -65,7 +65,7 @@ func (a *apiHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { } // create request and response - c := a.opts.Service.Client() + c := a.opts.Client req := c.NewRequest(service.Name, service.Endpoint.Name, request) rsp := &api.Response{} diff --git a/api/handler/event/event.go b/api/handler/event/event.go index 27393165..98dbfe90 100644 --- a/api/handler/event/event.go +++ b/api/handler/event/event.go @@ -118,7 +118,7 @@ func (e *event) ServeHTTP(w http.ResponseWriter, r *http.Request) { } // get client - c := e.opts.Service.Client() + c := e.opts.Client // create publication p := c.NewMessage(topic, ev) diff --git a/api/handler/options.go b/api/handler/options.go index 43401167..76f7eff5 100644 --- a/api/handler/options.go +++ b/api/handler/options.go @@ -1,8 +1,9 @@ package handler import ( - "github.com/micro/go-micro/v2" "github.com/micro/go-micro/v2/api/router" + "github.com/micro/go-micro/v2/client" + "github.com/micro/go-micro/v2/client/grpc" ) var ( @@ -13,7 +14,7 @@ type Options struct { MaxRecvSize int64 Namespace string Router router.Router - Service micro.Service + Client client.Client } type Option func(o *Options) @@ -25,9 +26,8 @@ func NewOptions(opts ...Option) Options { o(&options) } - // create service if its blank - if options.Service == nil { - WithService(micro.NewService())(&options) + if options.Client == nil { + WithClient(grpc.NewClient())(&options) } // set namespace if blank @@ -56,10 +56,9 @@ func WithRouter(r router.Router) Option { } } -// WithService specifies a micro.Service -func WithService(s micro.Service) Option { +func WithClient(c client.Client) Option { return func(o *Options) { - o.Service = s + o.Client = c } } diff --git a/api/handler/rpc/rpc.go b/api/handler/rpc/rpc.go index 7c06b663..d396274f 100644 --- a/api/handler/rpc/rpc.go +++ b/api/handler/rpc/rpc.go @@ -113,7 +113,7 @@ func (h *rpcHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { } // micro client - c := h.opts.Service.Client() + c := h.opts.Client // create context cx := ctx.FromRequest(r) From a056bdce7c292ba0cb0d5fe826a3342e2aee43e0 Mon Sep 17 00:00:00 2001 From: Asim Aslam Date: Sun, 12 Apr 2020 14:40:37 +0100 Subject: [PATCH 39/44] fix metadata parsing --- api/handler/rpc/rpc.go | 1 + 1 file changed, 1 insertion(+) diff --git a/api/handler/rpc/rpc.go b/api/handler/rpc/rpc.go index d396274f..58493c49 100644 --- a/api/handler/rpc/rpc.go +++ b/api/handler/rpc/rpc.go @@ -304,6 +304,7 @@ func requestPayload(r *http.Request) ([]byte, error) { // get fields from url path for k, v := range md { + k = strings.ToLower(k) // filter own keys if strings.HasPrefix(k, "x-api-field-") { matches[strings.TrimPrefix(k, "x-api-field-")] = v From 1bb6967a38ccaac050fd9be2eee6543be5783b0d Mon Sep 17 00:00:00 2001 From: Asim Aslam Date: Sun, 12 Apr 2020 23:41:21 +0100 Subject: [PATCH 40/44] reorder --- runtime/runtime.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/runtime/runtime.go b/runtime/runtime.go index f13db91b..0db14113 100644 --- a/runtime/runtime.go +++ b/runtime/runtime.go @@ -17,8 +17,6 @@ var ( // Runtime is a service runtime manager type Runtime interface { - // String describes runtime - String() string // Init initializes runtime Init(...Option) error // Create registers a service @@ -31,12 +29,14 @@ type Runtime interface { Delete(*Service) error // List the managed services List() ([]*Service, error) + // Logs returns the logs for a service + Logs(*Service, ...LogsOption) (LogStream, error) // Start starts the runtime Start() error // Stop shuts down the runtime Stop() error - // Logs - Logs(*Service, ...LogsOption) (LogStream, error) + // String describes runtime + String() string } // Stream returns a log stream From 5ef1698632d62926314f0cd2661befa1ac89837f Mon Sep 17 00:00:00 2001 From: Asim Aslam Date: Sun, 12 Apr 2020 23:43:55 +0100 Subject: [PATCH 41/44] remove readme --- runtime/README.md | 28 ---------------------------- 1 file changed, 28 deletions(-) delete mode 100644 runtime/README.md diff --git a/runtime/README.md b/runtime/README.md deleted file mode 100644 index 600bdceb..00000000 --- a/runtime/README.md +++ /dev/null @@ -1,28 +0,0 @@ -# Runtime - -A runtime for self governing services. - -## Overview - -In recent years we've started to develop complex architectures for the pipeline between writing code and running it. This -philosophy of build, run, manage or however many variations, has created a number of layers of abstraction that make it -all the more difficult to run code. - -Runtime manages the lifecycle of a service from source to running process. If the source is the *source of truth* then -everything in between running is wasted breath. Applications should be self governing and self sustaining. -To enable that we need libraries which make it possible. - -Runtime will fetch source code, build a binary and execute it. Any Go program that uses this library should be able -to run dependencies or itself with ease, with the ability to update itself as the source is updated. - -## Features - -- **Source** - Fetches source whether it be git, go, docker, etc -- **Package** - Compiles the source into a binary which can be executed -- **Process** - Executes a binary and creates a running process - -## Usage - -TODO - - From f840a5003ef482bcf9a0d25fd0ae3d9c0a722386 Mon Sep 17 00:00:00 2001 From: Asim Aslam Date: Sun, 12 Apr 2020 23:46:06 +0100 Subject: [PATCH 42/44] Remove runtime List --- runtime/runtime.go | 2 - runtime/service/proto/runtime.pb.go | 124 +++++++++++----------- runtime/service/proto/runtime.pb.micro.go | 19 +--- runtime/service/proto/runtime.proto | 1 - runtime/service/service.go | 22 ---- 5 files changed, 63 insertions(+), 105 deletions(-) diff --git a/runtime/runtime.go b/runtime/runtime.go index 0db14113..2df1530c 100644 --- a/runtime/runtime.go +++ b/runtime/runtime.go @@ -27,8 +27,6 @@ type Runtime interface { Update(*Service) error // Remove a service Delete(*Service) error - // List the managed services - List() ([]*Service, error) // Logs returns the logs for a service Logs(*Service, ...LogsOption) (LogStream, error) // Start starts the runtime diff --git a/runtime/service/proto/runtime.pb.go b/runtime/service/proto/runtime.pb.go index 2ac1af66..09a56e9f 100644 --- a/runtime/service/proto/runtime.pb.go +++ b/runtime/service/proto/runtime.pb.go @@ -1,5 +1,5 @@ // Code generated by protoc-gen-go. DO NOT EDIT. -// source: micro/go-micro/runtime/service/proto/runtime.proto +// source: github.com/micro/go-micro/runtime/service/proto/runtime.proto package go_micro_runtime @@ -38,7 +38,7 @@ func (m *Service) Reset() { *m = Service{} } func (m *Service) String() string { return proto.CompactTextString(m) } func (*Service) ProtoMessage() {} func (*Service) Descriptor() ([]byte, []int) { - return fileDescriptor_4bc91a8efec81434, []int{0} + return fileDescriptor_976fccef828ab1f0, []int{0} } func (m *Service) XXX_Unmarshal(b []byte) error { @@ -101,7 +101,7 @@ func (m *Event) Reset() { *m = Event{} } func (m *Event) String() string { return proto.CompactTextString(m) } func (*Event) ProtoMessage() {} func (*Event) Descriptor() ([]byte, []int) { - return fileDescriptor_4bc91a8efec81434, []int{1} + return fileDescriptor_976fccef828ab1f0, []int{1} } func (m *Event) XXX_Unmarshal(b []byte) error { @@ -172,7 +172,7 @@ func (m *CreateOptions) Reset() { *m = CreateOptions{} } func (m *CreateOptions) String() string { return proto.CompactTextString(m) } func (*CreateOptions) ProtoMessage() {} func (*CreateOptions) Descriptor() ([]byte, []int) { - return fileDescriptor_4bc91a8efec81434, []int{2} + return fileDescriptor_976fccef828ab1f0, []int{2} } func (m *CreateOptions) XXX_Unmarshal(b []byte) error { @@ -247,7 +247,7 @@ func (m *CreateRequest) Reset() { *m = CreateRequest{} } func (m *CreateRequest) String() string { return proto.CompactTextString(m) } func (*CreateRequest) ProtoMessage() {} func (*CreateRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_4bc91a8efec81434, []int{3} + return fileDescriptor_976fccef828ab1f0, []int{3} } func (m *CreateRequest) XXX_Unmarshal(b []byte) error { @@ -292,7 +292,7 @@ func (m *CreateResponse) Reset() { *m = CreateResponse{} } func (m *CreateResponse) String() string { return proto.CompactTextString(m) } func (*CreateResponse) ProtoMessage() {} func (*CreateResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_4bc91a8efec81434, []int{4} + return fileDescriptor_976fccef828ab1f0, []int{4} } func (m *CreateResponse) XXX_Unmarshal(b []byte) error { @@ -329,7 +329,7 @@ func (m *ReadOptions) Reset() { *m = ReadOptions{} } func (m *ReadOptions) String() string { return proto.CompactTextString(m) } func (*ReadOptions) ProtoMessage() {} func (*ReadOptions) Descriptor() ([]byte, []int) { - return fileDescriptor_4bc91a8efec81434, []int{5} + return fileDescriptor_976fccef828ab1f0, []int{5} } func (m *ReadOptions) XXX_Unmarshal(b []byte) error { @@ -382,7 +382,7 @@ func (m *ReadRequest) Reset() { *m = ReadRequest{} } func (m *ReadRequest) String() string { return proto.CompactTextString(m) } func (*ReadRequest) ProtoMessage() {} func (*ReadRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_4bc91a8efec81434, []int{6} + return fileDescriptor_976fccef828ab1f0, []int{6} } func (m *ReadRequest) XXX_Unmarshal(b []byte) error { @@ -421,7 +421,7 @@ func (m *ReadResponse) Reset() { *m = ReadResponse{} } func (m *ReadResponse) String() string { return proto.CompactTextString(m) } func (*ReadResponse) ProtoMessage() {} func (*ReadResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_4bc91a8efec81434, []int{7} + return fileDescriptor_976fccef828ab1f0, []int{7} } func (m *ReadResponse) XXX_Unmarshal(b []byte) error { @@ -460,7 +460,7 @@ func (m *DeleteRequest) Reset() { *m = DeleteRequest{} } func (m *DeleteRequest) String() string { return proto.CompactTextString(m) } func (*DeleteRequest) ProtoMessage() {} func (*DeleteRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_4bc91a8efec81434, []int{8} + return fileDescriptor_976fccef828ab1f0, []int{8} } func (m *DeleteRequest) XXX_Unmarshal(b []byte) error { @@ -498,7 +498,7 @@ func (m *DeleteResponse) Reset() { *m = DeleteResponse{} } func (m *DeleteResponse) String() string { return proto.CompactTextString(m) } func (*DeleteResponse) ProtoMessage() {} func (*DeleteResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_4bc91a8efec81434, []int{9} + return fileDescriptor_976fccef828ab1f0, []int{9} } func (m *DeleteResponse) XXX_Unmarshal(b []byte) error { @@ -530,7 +530,7 @@ func (m *UpdateRequest) Reset() { *m = UpdateRequest{} } func (m *UpdateRequest) String() string { return proto.CompactTextString(m) } func (*UpdateRequest) ProtoMessage() {} func (*UpdateRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_4bc91a8efec81434, []int{10} + return fileDescriptor_976fccef828ab1f0, []int{10} } func (m *UpdateRequest) XXX_Unmarshal(b []byte) error { @@ -568,7 +568,7 @@ func (m *UpdateResponse) Reset() { *m = UpdateResponse{} } func (m *UpdateResponse) String() string { return proto.CompactTextString(m) } func (*UpdateResponse) ProtoMessage() {} func (*UpdateResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_4bc91a8efec81434, []int{11} + return fileDescriptor_976fccef828ab1f0, []int{11} } func (m *UpdateResponse) XXX_Unmarshal(b []byte) error { @@ -599,7 +599,7 @@ func (m *ListRequest) Reset() { *m = ListRequest{} } func (m *ListRequest) String() string { return proto.CompactTextString(m) } func (*ListRequest) ProtoMessage() {} func (*ListRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_4bc91a8efec81434, []int{12} + return fileDescriptor_976fccef828ab1f0, []int{12} } func (m *ListRequest) XXX_Unmarshal(b []byte) error { @@ -631,7 +631,7 @@ func (m *ListResponse) Reset() { *m = ListResponse{} } func (m *ListResponse) String() string { return proto.CompactTextString(m) } func (*ListResponse) ProtoMessage() {} func (*ListResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_4bc91a8efec81434, []int{13} + return fileDescriptor_976fccef828ab1f0, []int{13} } func (m *ListResponse) XXX_Unmarshal(b []byte) error { @@ -679,7 +679,7 @@ func (m *LogsRequest) Reset() { *m = LogsRequest{} } func (m *LogsRequest) String() string { return proto.CompactTextString(m) } func (*LogsRequest) ProtoMessage() {} func (*LogsRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_4bc91a8efec81434, []int{14} + return fileDescriptor_976fccef828ab1f0, []int{14} } func (m *LogsRequest) XXX_Unmarshal(b []byte) error { @@ -744,7 +744,7 @@ func (m *LogRecord) Reset() { *m = LogRecord{} } func (m *LogRecord) String() string { return proto.CompactTextString(m) } func (*LogRecord) ProtoMessage() {} func (*LogRecord) Descriptor() ([]byte, []int) { - return fileDescriptor_4bc91a8efec81434, []int{15} + return fileDescriptor_976fccef828ab1f0, []int{15} } func (m *LogRecord) XXX_Unmarshal(b []byte) error { @@ -808,51 +808,51 @@ func init() { } func init() { - proto.RegisterFile("micro/go-micro/runtime/service/proto/runtime.proto", fileDescriptor_4bc91a8efec81434) + proto.RegisterFile("github.com/micro/go-micro/runtime/service/proto/runtime.proto", fileDescriptor_976fccef828ab1f0) } -var fileDescriptor_4bc91a8efec81434 = []byte{ - // 663 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xa4, 0x55, 0xcb, 0x6e, 0xd3, 0x40, - 0x14, 0xad, 0xe3, 0x3c, 0xda, 0x6b, 0x82, 0xaa, 0x51, 0x85, 0x4c, 0x79, 0x45, 0xde, 0x50, 0x16, - 0xb8, 0x28, 0x15, 0xe2, 0xb5, 0x6c, 0x53, 0x36, 0x8d, 0x90, 0x8c, 0xfa, 0x01, 0x83, 0x73, 0x65, - 0x59, 0xad, 0x3d, 0xc6, 0x33, 0x8e, 0x94, 0x15, 0xdf, 0xc0, 0x57, 0xb1, 0x85, 0x3f, 0x42, 0xf3, - 0xf0, 0x2b, 0xb1, 0xbb, 0xc9, 0x6e, 0xee, 0xe4, 0xce, 0xf1, 0x39, 0x67, 0xce, 0x9d, 0xc0, 0x3c, - 0x89, 0xc3, 0x9c, 0x9d, 0x47, 0xec, 0xad, 0x5e, 0xe4, 0x45, 0x2a, 0xe2, 0x04, 0xcf, 0x39, 0xe6, - 0xeb, 0x38, 0xc4, 0xf3, 0x2c, 0x67, 0xa2, 0xda, 0xf5, 0x55, 0x45, 0x8e, 0x23, 0xe6, 0xab, 0x6e, - 0xdf, 0xec, 0x7b, 0xff, 0x2c, 0x98, 0x7c, 0xd7, 0x27, 0x08, 0x81, 0x61, 0x4a, 0x13, 0x74, 0xad, - 0x99, 0x75, 0x76, 0x14, 0xa8, 0x35, 0x71, 0x61, 0xb2, 0xc6, 0x9c, 0xc7, 0x2c, 0x75, 0x07, 0x6a, - 0xbb, 0x2c, 0xc9, 0x13, 0x18, 0x73, 0x56, 0xe4, 0x21, 0xba, 0xb6, 0xfa, 0xc1, 0x54, 0xe4, 0x12, - 0x0e, 0x13, 0x14, 0x74, 0x45, 0x05, 0x75, 0x87, 0x33, 0xfb, 0xcc, 0x99, 0xbf, 0xf6, 0xb7, 0x3f, - 0xeb, 0x9b, 0x4f, 0xfa, 0x4b, 0xd3, 0xb9, 0x48, 0x45, 0xbe, 0x09, 0xaa, 0x83, 0xa7, 0x5f, 0x60, - 0xda, 0xfa, 0x89, 0x1c, 0x83, 0x7d, 0x87, 0x1b, 0x43, 0x4d, 0x2e, 0xc9, 0x09, 0x8c, 0xd6, 0xf4, - 0xbe, 0x40, 0xc3, 0x4b, 0x17, 0x9f, 0x07, 0x1f, 0x2d, 0x2f, 0x81, 0xd1, 0x62, 0x8d, 0xa9, 0x90, - 0x82, 0xc4, 0x26, 0xab, 0x04, 0xc9, 0x35, 0x79, 0x0e, 0x47, 0x92, 0x01, 0x17, 0x34, 0xc9, 0xd4, - 0x51, 0x3b, 0xa8, 0x37, 0xa4, 0x5c, 0xe3, 0x9f, 0x51, 0x55, 0x96, 0x4d, 0x23, 0x86, 0x2d, 0x23, - 0xbc, 0xdf, 0x16, 0x4c, 0x2f, 0x73, 0xa4, 0x02, 0xbf, 0x65, 0x22, 0x66, 0x29, 0x97, 0xbd, 0x21, - 0x4b, 0x12, 0x9a, 0xae, 0x5c, 0x6b, 0x66, 0xcb, 0x5e, 0x53, 0x4a, 0x46, 0x34, 0x8f, 0xb8, 0x3b, - 0x50, 0xdb, 0x6a, 0x2d, 0xa5, 0x61, 0xba, 0x76, 0x6d, 0xb5, 0x25, 0x97, 0xd2, 0x5a, 0x56, 0x88, - 0xac, 0x10, 0xe6, 0x53, 0xa6, 0xaa, 0xf4, 0x8c, 0x1a, 0x7a, 0x4e, 0x60, 0x14, 0x27, 0x34, 0x42, - 0x77, 0xac, 0x6d, 0x50, 0x85, 0xf7, 0xab, 0xa4, 0x14, 0xe0, 0xcf, 0x02, 0xb9, 0x20, 0x17, 0xb5, - 0x30, 0xe9, 0x86, 0x33, 0x7f, 0xda, 0x7b, 0x29, 0xb5, 0xe6, 0x4f, 0x30, 0x61, 0x5a, 0x92, 0x72, - 0xca, 0x99, 0xbf, 0xda, 0x3d, 0xd4, 0x52, 0x1e, 0x94, 0xfd, 0xde, 0x31, 0x3c, 0x2e, 0x09, 0xf0, - 0x8c, 0xa5, 0x1c, 0xbd, 0x5b, 0x70, 0x02, 0xa4, 0xab, 0x86, 0x47, 0x4d, 0x42, 0xdd, 0x4e, 0x6f, - 0x45, 0xae, 0xd4, 0x6f, 0xd7, 0xfa, 0xbd, 0x6b, 0x0d, 0x5b, 0xea, 0xfc, 0x50, 0x53, 0xd6, 0x3a, - 0x5f, 0xec, 0x52, 0x6e, 0xd0, 0xa8, 0x09, 0x2f, 0xe0, 0x91, 0xc6, 0xd1, 0x74, 0xc9, 0x7b, 0x38, - 0x34, 0x84, 0xb8, 0xba, 0xc4, 0x07, 0x1d, 0xab, 0x5a, 0xbd, 0x2b, 0x98, 0x5e, 0xe1, 0x3d, 0xee, - 0x67, 0xbc, 0x74, 0xaf, 0x44, 0x31, 0xee, 0x5d, 0xc1, 0xf4, 0x36, 0x5b, 0xd1, 0xfd, 0x71, 0x4b, - 0x14, 0x83, 0x3b, 0x05, 0xe7, 0x26, 0xe6, 0xc2, 0xa0, 0x4a, 0x17, 0x74, 0xb9, 0x9f, 0x0b, 0x77, - 0xe0, 0xdc, 0xb0, 0x88, 0x97, 0x5c, 0xfb, 0xef, 0x5a, 0x3e, 0x22, 0x22, 0x47, 0x9a, 0xa8, 0xab, - 0x3e, 0x0c, 0x4c, 0x25, 0x53, 0x1d, 0xb2, 0x22, 0x15, 0xea, 0xaa, 0xed, 0x40, 0x17, 0x72, 0x97, - 0xc7, 0x69, 0x88, 0x6a, 0x2c, 0xec, 0x40, 0x17, 0xde, 0x1f, 0x0b, 0x8e, 0x6e, 0x58, 0x14, 0x60, - 0xc8, 0xf2, 0x55, 0x7b, 0xbe, 0xad, 0xed, 0xf9, 0x5e, 0x34, 0x1e, 0xa7, 0x81, 0xd2, 0xf3, 0x66, - 0x57, 0x4f, 0x05, 0xd6, 0xf7, 0x3c, 0x49, 0x41, 0x09, 0x72, 0x2e, 0xc7, 0xce, 0x3c, 0x13, 0xa6, - 0xdc, 0xeb, 0xe1, 0x9a, 0xff, 0xb5, 0x61, 0x12, 0x68, 0x12, 0x64, 0x09, 0x63, 0x3d, 0x40, 0xa4, - 0x77, 0xe8, 0x8c, 0xbd, 0xa7, 0xb3, 0xfe, 0x06, 0x73, 0xcb, 0x07, 0xe4, 0x2b, 0x0c, 0x65, 0xbc, - 0x49, 0xcf, 0x38, 0x94, 0x50, 0x2f, 0xfb, 0x7e, 0xae, 0x80, 0x96, 0x30, 0xd6, 0xd1, 0xec, 0xe2, - 0xd5, 0x8a, 0x7e, 0x17, 0xaf, 0xad, 0x54, 0x2b, 0x38, 0x9d, 0xc8, 0x2e, 0xb8, 0x56, 0xe2, 0xbb, - 0xe0, 0xb6, 0xc2, 0xac, 0x64, 0xca, 0xfc, 0x76, 0xc9, 0x6c, 0xc4, 0xbc, 0x4b, 0x66, 0x33, 0xf6, - 0xde, 0x01, 0xb9, 0x86, 0xa1, 0x4c, 0x70, 0x27, 0x50, 0x9d, 0xec, 0xd3, 0x67, 0x0f, 0xa4, 0xc7, - 0x3b, 0x78, 0x67, 0xfd, 0x18, 0xab, 0x3f, 0xde, 0x8b, 0xff, 0x01, 0x00, 0x00, 0xff, 0xff, 0x17, - 0xe1, 0xab, 0x77, 0xae, 0x07, 0x00, 0x00, +var fileDescriptor_976fccef828ab1f0 = []byte{ + // 662 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xa4, 0x55, 0xbb, 0x6e, 0xdb, 0x4a, + 0x10, 0x35, 0x45, 0x3d, 0xec, 0xd1, 0xd5, 0x85, 0xb1, 0x30, 0x02, 0xc6, 0x79, 0x09, 0x6c, 0xe2, + 0x14, 0xa1, 0x02, 0x19, 0x41, 0x5e, 0x48, 0x65, 0xcb, 0x69, 0x6c, 0x04, 0x60, 0xe0, 0x0f, 0x58, + 0x53, 0x03, 0x86, 0xb0, 0x97, 0xcb, 0x70, 0x97, 0x02, 0x5c, 0xa5, 0x4c, 0x9d, 0xaf, 0x4a, 0x9d, + 0x3f, 0x0a, 0xf6, 0x41, 0x8a, 0x94, 0x48, 0x37, 0xea, 0x76, 0x46, 0xb3, 0x87, 0xe7, 0x9c, 0x99, + 0x59, 0xc1, 0xe7, 0x38, 0x91, 0xdf, 0x8b, 0x9b, 0x20, 0xe2, 0x6c, 0xc6, 0x92, 0x28, 0xe7, 0xb3, + 0x98, 0xbf, 0x36, 0x87, 0xbc, 0x48, 0x65, 0xc2, 0x70, 0x26, 0x30, 0x5f, 0x25, 0x11, 0xce, 0xb2, + 0x9c, 0xcb, 0x2a, 0x1b, 0xe8, 0x88, 0x1c, 0xc6, 0x3c, 0xd0, 0xd5, 0x81, 0xcd, 0xfb, 0x7f, 0x1d, + 0x18, 0x7d, 0x33, 0x37, 0x08, 0x81, 0x7e, 0x4a, 0x19, 0x7a, 0xce, 0xd4, 0x39, 0x39, 0x08, 0xf5, + 0x99, 0x78, 0x30, 0x5a, 0x61, 0x2e, 0x12, 0x9e, 0x7a, 0x3d, 0x9d, 0x2e, 0x43, 0xf2, 0x08, 0x86, + 0x82, 0x17, 0x79, 0x84, 0x9e, 0xab, 0x7f, 0xb0, 0x11, 0x39, 0x83, 0x7d, 0x86, 0x92, 0x2e, 0xa9, + 0xa4, 0x5e, 0x7f, 0xea, 0x9e, 0x8c, 0xe7, 0x2f, 0x83, 0xcd, 0xcf, 0x06, 0xf6, 0x93, 0xc1, 0x95, + 0xad, 0x5c, 0xa4, 0x32, 0xbf, 0x0f, 0xab, 0x8b, 0xc7, 0x9f, 0x60, 0xd2, 0xf8, 0x89, 0x1c, 0x82, + 0x7b, 0x8b, 0xf7, 0x96, 0x9a, 0x3a, 0x92, 0x23, 0x18, 0xac, 0xe8, 0x5d, 0x81, 0x96, 0x97, 0x09, + 0x3e, 0xf6, 0xde, 0x3b, 0x3e, 0x83, 0xc1, 0x62, 0x85, 0xa9, 0x54, 0x82, 0xe4, 0x7d, 0x56, 0x09, + 0x52, 0x67, 0xf2, 0x14, 0x0e, 0x14, 0x03, 0x21, 0x29, 0xcb, 0xf4, 0x55, 0x37, 0x5c, 0x27, 0x94, + 0x5c, 0xeb, 0x9f, 0x55, 0x55, 0x86, 0x75, 0x23, 0xfa, 0x0d, 0x23, 0xfc, 0xdf, 0x0e, 0x4c, 0xce, + 0x72, 0xa4, 0x12, 0xbf, 0x66, 0x32, 0xe1, 0xa9, 0x50, 0xb5, 0x11, 0x67, 0x8c, 0xa6, 0x4b, 0xcf, + 0x99, 0xba, 0xaa, 0xd6, 0x86, 0x8a, 0x11, 0xcd, 0x63, 0xe1, 0xf5, 0x74, 0x5a, 0x9f, 0x95, 0x34, + 0x4c, 0x57, 0x9e, 0xab, 0x53, 0xea, 0xa8, 0xac, 0xe5, 0x85, 0xcc, 0x0a, 0x69, 0x3f, 0x65, 0xa3, + 0x4a, 0xcf, 0xa0, 0xa6, 0xe7, 0x08, 0x06, 0x09, 0xa3, 0x31, 0x7a, 0x43, 0x63, 0x83, 0x0e, 0xfc, + 0x9f, 0x25, 0xa5, 0x10, 0x7f, 0x14, 0x28, 0x24, 0x39, 0x5d, 0x0b, 0x53, 0x6e, 0x8c, 0xe7, 0x8f, + 0x3b, 0x9b, 0xb2, 0xd6, 0xfc, 0x01, 0x46, 0xdc, 0x48, 0xd2, 0x4e, 0x8d, 0xe7, 0x2f, 0xb6, 0x2f, + 0x35, 0x94, 0x87, 0x65, 0xbd, 0x7f, 0x08, 0xff, 0x97, 0x04, 0x44, 0xc6, 0x53, 0x81, 0xfe, 0x35, + 0x8c, 0x43, 0xa4, 0xcb, 0x9a, 0x47, 0x75, 0x42, 0xed, 0x4e, 0x6f, 0x8c, 0x5c, 0xa9, 0xdf, 0x5d, + 0xeb, 0xf7, 0x2f, 0x0c, 0x6c, 0xa9, 0xf3, 0xdd, 0x9a, 0xb2, 0xd1, 0xf9, 0x6c, 0x9b, 0x72, 0x8d, + 0xc6, 0x9a, 0xf0, 0x02, 0xfe, 0x33, 0x38, 0x86, 0x2e, 0x79, 0x0b, 0xfb, 0x96, 0x90, 0xd0, 0x4d, + 0x7c, 0xd0, 0xb1, 0xaa, 0xd4, 0x3f, 0x87, 0xc9, 0x39, 0xde, 0xe1, 0x6e, 0xc6, 0x2b, 0xf7, 0x4a, + 0x14, 0xeb, 0xde, 0x39, 0x4c, 0xae, 0xb3, 0x25, 0xdd, 0x1d, 0xb7, 0x44, 0xb1, 0xb8, 0x13, 0x18, + 0x5f, 0x26, 0x42, 0x5a, 0x54, 0xe5, 0x82, 0x09, 0x77, 0x73, 0xe1, 0x16, 0xc6, 0x97, 0x3c, 0x16, + 0x25, 0xd7, 0xee, 0x5e, 0xab, 0x47, 0x44, 0xe6, 0x48, 0x99, 0x6e, 0xf5, 0x7e, 0x68, 0x23, 0x35, + 0xd5, 0x11, 0x2f, 0x52, 0xa9, 0x5b, 0xed, 0x86, 0x26, 0x50, 0x59, 0x91, 0xa4, 0x11, 0xea, 0xb5, + 0x70, 0x43, 0x13, 0xf8, 0x7f, 0x1c, 0x38, 0xb8, 0xe4, 0x71, 0x88, 0x11, 0xcf, 0x97, 0xcd, 0xfd, + 0x76, 0x36, 0xf7, 0x7b, 0x51, 0x7b, 0x9c, 0x7a, 0x5a, 0xcf, 0xab, 0x6d, 0x3d, 0x15, 0x58, 0xd7, + 0xf3, 0xa4, 0x04, 0x31, 0x14, 0x42, 0xad, 0x9d, 0x7d, 0x26, 0x6c, 0xb8, 0xd3, 0xc3, 0x35, 0xff, + 0xe5, 0xc2, 0x28, 0x34, 0x24, 0xc8, 0x15, 0x0c, 0xcd, 0x02, 0x91, 0xce, 0xa5, 0xb3, 0xf6, 0x1e, + 0x4f, 0xbb, 0x0b, 0x6c, 0x97, 0xf7, 0xc8, 0x17, 0xe8, 0xab, 0xf1, 0x26, 0x1d, 0xeb, 0x50, 0x42, + 0x3d, 0xef, 0xfa, 0xb9, 0x02, 0xba, 0x82, 0xa1, 0x19, 0xcd, 0x36, 0x5e, 0x8d, 0xd1, 0x6f, 0xe3, + 0xb5, 0x31, 0xd5, 0x1a, 0xce, 0x4c, 0x64, 0x1b, 0x5c, 0x63, 0xe2, 0xdb, 0xe0, 0x36, 0x86, 0x79, + 0x8f, 0x5c, 0x40, 0x5f, 0x0d, 0x5e, 0x9b, 0xcc, 0xda, 0x40, 0x1e, 0x3f, 0x79, 0xa0, 0xe9, 0xfe, + 0xde, 0x1b, 0xe7, 0x66, 0xa8, 0xff, 0x2f, 0x4f, 0xff, 0x05, 0x00, 0x00, 0xff, 0xff, 0x40, 0x42, + 0xb3, 0x4e, 0x70, 0x07, 0x00, 0x00, } diff --git a/runtime/service/proto/runtime.pb.micro.go b/runtime/service/proto/runtime.pb.micro.go index eff7c76e..da23a4e7 100644 --- a/runtime/service/proto/runtime.pb.micro.go +++ b/runtime/service/proto/runtime.pb.micro.go @@ -1,5 +1,5 @@ // Code generated by protoc-gen-micro. DO NOT EDIT. -// source: micro/go-micro/runtime/service/proto/runtime.proto +// source: github.com/micro/go-micro/runtime/service/proto/runtime.proto package go_micro_runtime @@ -38,7 +38,6 @@ type RuntimeService interface { Read(ctx context.Context, in *ReadRequest, opts ...client.CallOption) (*ReadResponse, error) Delete(ctx context.Context, in *DeleteRequest, opts ...client.CallOption) (*DeleteResponse, error) Update(ctx context.Context, in *UpdateRequest, opts ...client.CallOption) (*UpdateResponse, error) - List(ctx context.Context, in *ListRequest, opts ...client.CallOption) (*ListResponse, error) Logs(ctx context.Context, in *LogsRequest, opts ...client.CallOption) (Runtime_LogsService, error) } @@ -94,16 +93,6 @@ func (c *runtimeService) Update(ctx context.Context, in *UpdateRequest, opts ... return out, nil } -func (c *runtimeService) List(ctx context.Context, in *ListRequest, opts ...client.CallOption) (*ListResponse, error) { - req := c.c.NewRequest(c.name, "Runtime.List", in) - out := new(ListResponse) - err := c.c.Call(ctx, req, out, opts...) - if err != nil { - return nil, err - } - return out, nil -} - func (c *runtimeService) Logs(ctx context.Context, in *LogsRequest, opts ...client.CallOption) (Runtime_LogsService, error) { req := c.c.NewRequest(c.name, "Runtime.Logs", &LogsRequest{}) stream, err := c.c.Stream(ctx, req, opts...) @@ -160,7 +149,6 @@ type RuntimeHandler interface { Read(context.Context, *ReadRequest, *ReadResponse) error Delete(context.Context, *DeleteRequest, *DeleteResponse) error Update(context.Context, *UpdateRequest, *UpdateResponse) error - List(context.Context, *ListRequest, *ListResponse) error Logs(context.Context, *LogsRequest, Runtime_LogsStream) error } @@ -170,7 +158,6 @@ func RegisterRuntimeHandler(s server.Server, hdlr RuntimeHandler, opts ...server Read(ctx context.Context, in *ReadRequest, out *ReadResponse) error Delete(ctx context.Context, in *DeleteRequest, out *DeleteResponse) error Update(ctx context.Context, in *UpdateRequest, out *UpdateResponse) error - List(ctx context.Context, in *ListRequest, out *ListResponse) error Logs(ctx context.Context, stream server.Stream) error } type Runtime struct { @@ -200,10 +187,6 @@ func (h *runtimeHandler) Update(ctx context.Context, in *UpdateRequest, out *Upd return h.RuntimeHandler.Update(ctx, in, out) } -func (h *runtimeHandler) List(ctx context.Context, in *ListRequest, out *ListResponse) error { - return h.RuntimeHandler.List(ctx, in, out) -} - func (h *runtimeHandler) Logs(ctx context.Context, stream server.Stream) error { m := new(LogsRequest) if err := stream.Recv(m); err != nil { diff --git a/runtime/service/proto/runtime.proto b/runtime/service/proto/runtime.proto index e8d563e0..65c2c497 100644 --- a/runtime/service/proto/runtime.proto +++ b/runtime/service/proto/runtime.proto @@ -7,7 +7,6 @@ service Runtime { rpc Read(ReadRequest) returns (ReadResponse) {}; rpc Delete(DeleteRequest) returns (DeleteResponse) {}; rpc Update(UpdateRequest) returns (UpdateResponse) {}; - rpc List(ListRequest) returns (ListResponse) {}; rpc Logs(LogsRequest) returns (stream LogRecord) {}; } diff --git a/runtime/service/service.go b/runtime/service/service.go index 8cadfcdf..d065038c 100644 --- a/runtime/service/service.go +++ b/runtime/service/service.go @@ -194,28 +194,6 @@ func (s *svc) Delete(svc *runtime.Service) error { return nil } -// List lists all services managed by the runtime -func (s *svc) List() ([]*runtime.Service, error) { - // list all services managed by the runtime - resp, err := s.runtime.List(context.Background(), &pb.ListRequest{}) - if err != nil { - return nil, err - } - - services := make([]*runtime.Service, 0, len(resp.Services)) - for _, service := range resp.Services { - svc := &runtime.Service{ - Name: service.Name, - Version: service.Version, - Source: service.Source, - Metadata: service.Metadata, - } - services = append(services, svc) - } - - return services, nil -} - // Start starts the runtime func (s *svc) Start() error { // NOTE: nothing to be done here From 4bdc18d64aacf95b58cfb53113bcf8b17b32aa86 Mon Sep 17 00:00:00 2001 From: Asim Aslam Date: Mon, 13 Apr 2020 22:15:21 +0100 Subject: [PATCH 43/44] Update README.md --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index d6b39afc..a39364a1 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # Go Micro [![License](https://img.shields.io/:license-apache-blue.svg)](https://opensource.org/licenses/Apache-2.0) [![Go.Dev reference](https://img.shields.io/badge/go.dev-reference-007d9c?logo=go&logoColor=white&style=flat-square)](https://pkg.go.dev/github.com/micro/go-micro?tab=doc) [![Travis CI](https://api.travis-ci.org/micro/go-micro.svg?branch=master)](https://travis-ci.org/micro/go-micro) [![Go Report Card](https://goreportcard.com/badge/micro/go-micro)](https://goreportcard.com/report/github.com/micro/go-micro) -Go Micro is a framework for microservice development. +Go Micro is a framework for distributed systems development. ## Overview From e515005083ef83dc1978359c78fd16f10c3a5f9b Mon Sep 17 00:00:00 2001 From: Asim Aslam Date: Mon, 13 Apr 2020 23:05:39 +0100 Subject: [PATCH 44/44] Remove only allowing certain methods --- api/handler/rpc/rpc.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/api/handler/rpc/rpc.go b/api/handler/rpc/rpc.go index 58493c49..d9ff31ad 100644 --- a/api/handler/rpc/rpc.go +++ b/api/handler/rpc/rpc.go @@ -100,11 +100,6 @@ func (h *rpcHandler) ServeHTTP(w http.ResponseWriter, r *http.Request) { return } - if h.opts.Router == nil && r.Method != "GET" { - writeError(w, r, errors.MethodNotAllowed("go.micro.api", "method not allowed")) - return - } - ct := r.Header.Get("Content-Type") // Strip charset from Content-Type (like `application/json; charset=UTF-8`)