From 1599d717af675fc9ba5bcdf75a4c4a9ab62b027a Mon Sep 17 00:00:00 2001 From: Shulhan Date: Wed, 14 Mar 2018 18:51:38 +0700 Subject: [PATCH 1/6] Add option to enable TCP check with Consul registry One disadvantage of using TTL based health check is the high network traffic between Consul agent (either between servers, or between server and client). In order for the services considered alive by Consul, microservices must send an update TTL to Consul every n seconds (currently 30 seconds). Here is the explanation about TTL check from Consul documentation [1] Time to Live (TTL) - These checks retain their last known state for a given TTL. The state of the check must be updated periodically over the HTTP interface. If an external system fails to update the status within a given TTL, the check is set to the failed state. This mechanism, conceptually similar to a dead man's switch, relies on the application to directly report its health. For example, a healthy app can periodically PUT a status update to the HTTP endpoint; if the app fails, the TTL will expire and the health check enters a critical state. The endpoints used to update health information for a given check are the pass endpoint and the fail endpoint. TTL checks also persist their last known status to disk. This allows the Consul agent to restore the last known status of the check across restarts. Persisted check status is valid through the end of the TTL from the time of the last check. Hint: TTL checks also persist their last known status to disk. This allows the Consul agent to restore the last known status of the check across restarts. When microservices update the TTL, Consul will write to disk. Writing to disk means all other slaves need to replicate it, which means master need to inform other standby Consul to pull the new catalog. Hence, the increased traffic. More information about this issue can be viewed at Consul mailing list [2]. [1] https://www.consul.io/docs/agent/checks.html [2] https://groups.google.com/forum/#!topic/consul-tool/84h7qmCCpjg --- registry/consul_registry.go | 32 ++++++++++++++++++++++++-------- registry/options.go | 21 ++++++++++++++++++++- server/options.go | 21 ++++++++++++++++++++- server/rpc_server.go | 5 ++++- 4 files changed, 68 insertions(+), 11 deletions(-) diff --git a/registry/consul_registry.go b/registry/consul_registry.go index a3387dba..366b77fd 100644 --- a/registry/consul_registry.go +++ b/registry/consul_registry.go @@ -115,6 +115,19 @@ func (c *consulRegistry) Deregister(s *Service) error { return c.Client.Agent().ServiceDeregister(node.Id) } +func getDeregisterTTL(t time.Duration) time.Duration { + // splay slightly for the watcher? + splay := time.Second * 5 + deregTTL := t + splay + + // consul has a minimum timeout on deregistration of 1 minute. + if t < time.Minute { + deregTTL = time.Minute + splay + } + + return deregTTL +} + func (c *consulRegistry) Register(s *Service, opts ...RegisterOption) error { if len(s.Nodes) == 0 { return errors.New("Require at least one node") @@ -155,16 +168,19 @@ func (c *consulRegistry) Register(s *Service, opts ...RegisterOption) error { var check *consul.AgentServiceCheck - // if the TTL is greater than 0 create an associated check - if options.TTL > time.Duration(0) { - // splay slightly for the watcher? - splay := time.Second * 5 - deregTTL := options.TTL + splay - // consul has a minimum timeout on deregistration of 1 minute. - if options.TTL < time.Minute { - deregTTL = time.Minute + splay + if options.TCPCheck { + deregTTL := getDeregisterTTL(options.Interval) + + check = &consul.AgentServiceCheck{ + TCP: c.Address, + Interval: fmt.Sprintf("%v", options.Interval), + DeregisterCriticalServiceAfter: fmt.Sprintf("%v", deregTTL), } + // if the TTL is greater than 0 create an associated check + } else if options.TTL > time.Duration(0) { + deregTTL := getDeregisterTTL(options.TTL) + check = &consul.AgentServiceCheck{ TTL: fmt.Sprintf("%v", options.TTL), DeregisterCriticalServiceAfter: fmt.Sprintf("%v", deregTTL), diff --git a/registry/options.go b/registry/options.go index aa9c3f4f..086b599d 100644 --- a/registry/options.go +++ b/registry/options.go @@ -18,7 +18,9 @@ type Options struct { } type RegisterOptions struct { - TTL time.Duration + TCPCheck bool + TTL time.Duration + Interval time.Duration // Other options for implementations of the interface // can be stored in a context Context context.Context @@ -66,6 +68,23 @@ func RegisterTTL(t time.Duration) RegisterOption { } } +// +// RegisterTCPCheck will tell the service provider to check the service address +// and port every `t` interval. It will enabled only if `t` is greater than 0. +// This option is for registry using Consul, see `TCP + Interval` more +// information [1]. +// +// [1] https://www.consul.io/docs/agent/checks.html +// +func RegisterTCPCheck(t time.Duration) RegisterOption { + return func(o *RegisterOptions) { + if t > time.Duration(0) { + o.TCPCheck = true + o.Interval = t + } + } +} + // Watch a service func WatchService(name string) WatchOption { return func(o *WatchOptions) { diff --git a/server/options.go b/server/options.go index 5b9aa98e..5fd4e4bb 100644 --- a/server/options.go +++ b/server/options.go @@ -25,7 +25,9 @@ type Options struct { HdlrWrappers []HandlerWrapper SubWrappers []SubscriberWrapper - RegisterTTL time.Duration + RegisterTCPCheck bool + RegisterTTL time.Duration + RegisterInterval time.Duration // Debug Handler which can be set by a user DebugHandler debug.DebugHandler @@ -164,6 +166,23 @@ func RegisterTTL(t time.Duration) Option { } } +// +// RegisterTCPCheck will tell the service provider to check the service address +// and port every `t` interval. It will enabled only if `t` is greater than 0. +// This option is for registry using Consul, see `TCP + Interval` more +// information [1]. +// +// [1] https://www.consul.io/docs/agent/checks.html +// +func RegisterTCPCheck(t time.Duration) Option { + return func(o *Options) { + if t > time.Duration(0) { + o.RegisterTCPCheck = true + o.RegisterInterval = t + } + } +} + // Wait tells the server to wait for requests to finish before exiting func Wait(b bool) Option { return func(o *Options) { diff --git a/server/rpc_server.go b/server/rpc_server.go index aa1285d9..f1ec257b 100644 --- a/server/rpc_server.go +++ b/server/rpc_server.go @@ -278,7 +278,10 @@ func (s *rpcServer) Register() error { } // create registry options - rOpts := []registry.RegisterOption{registry.RegisterTTL(config.RegisterTTL)} + rOpts := []registry.RegisterOption{ + registry.RegisterTTL(config.RegisterTTL), + registry.RegisterTCPCheck(config.RegisterInterval), + } if err := config.Registry.Register(service, rOpts...); err != nil { return err From f4cdfaf27f02bc8f4ce158e14841060a735d613d Mon Sep 17 00:00:00 2001 From: Shulhan Date: Mon, 19 Mar 2018 20:34:12 +0700 Subject: [PATCH 2/6] Fix TCP address and port on service check registration --- registry/consul_registry.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/registry/consul_registry.go b/registry/consul_registry.go index 366b77fd..54ca80f5 100644 --- a/registry/consul_registry.go +++ b/registry/consul_registry.go @@ -172,7 +172,7 @@ func (c *consulRegistry) Register(s *Service, opts ...RegisterOption) error { deregTTL := getDeregisterTTL(options.Interval) check = &consul.AgentServiceCheck{ - TCP: c.Address, + TCP: fmt.Sprintf("%s:%d", node.Address, node.Port), Interval: fmt.Sprintf("%v", options.Interval), DeregisterCriticalServiceAfter: fmt.Sprintf("%v", deregTTL), } From 68ab671bd0fc2b4c9b1e45c1681dd8eabe088f5d Mon Sep 17 00:00:00 2001 From: Shulhan Date: Mon, 19 Mar 2018 20:34:56 +0700 Subject: [PATCH 3/6] Use registry.options.Context to set Consul TCP check option --- registry/consul/options.go | 21 +++++++++++++++++++++ registry/consul_registry.go | 13 +++++++++++++ registry/options.go | 17 ----------------- server/options.go | 21 +-------------------- server/rpc_server.go | 5 +---- 5 files changed, 36 insertions(+), 41 deletions(-) diff --git a/registry/consul/options.go b/registry/consul/options.go index aefce92a..8957657c 100644 --- a/registry/consul/options.go +++ b/registry/consul/options.go @@ -2,6 +2,7 @@ package consul import ( "context" + "time" consul "github.com/hashicorp/consul/api" "github.com/micro/go-micro/registry" @@ -15,3 +16,23 @@ func Config(c *consul.Config) registry.Option { o.Context = context.WithValue(o.Context, "consul_config", c) } } + +// +// RegisterTCPCheck will tell the service provider to check the service address +// and port every `t` interval. It will enabled only if `t` is greater than 0. +// See `TCP + Interval` for more information [1]. +// +// [1] https://www.consul.io/docs/agent/checks.html +// +func RegisterTCPCheck(t time.Duration) registry.Option { + return func(o *registry.Options) { + if t <= time.Duration(0) { + return + } + if o.Context == nil { + o.Context = context.Background() + } + o.Context = context.WithValue(o.Context, + registry.ConsulRegisterTCPCheckKey, t) + } +} diff --git a/registry/consul_registry.go b/registry/consul_registry.go index 54ca80f5..dda716a4 100644 --- a/registry/consul_registry.go +++ b/registry/consul_registry.go @@ -14,6 +14,10 @@ import ( hash "github.com/mitchellh/hashstructure" ) +const ( + ConsulRegisterTCPCheckKey = "consul_register_tcp_check" +) + type consulRegistry struct { Address string Client *consul.Client @@ -138,6 +142,15 @@ func (c *consulRegistry) Register(s *Service, opts ...RegisterOption) error { o(&options) } + if c.opts.Context != nil { + tcpCheckInterval, ok := c.opts.Context. + Value(ConsulRegisterTCPCheckKey).(time.Duration) + if ok { + options.TCPCheck = true + options.Interval = tcpCheckInterval + } + } + // create hash of service; uint64 h, err := hash.Hash(s, nil) if err != nil { diff --git a/registry/options.go b/registry/options.go index 086b599d..cb687834 100644 --- a/registry/options.go +++ b/registry/options.go @@ -68,23 +68,6 @@ func RegisterTTL(t time.Duration) RegisterOption { } } -// -// RegisterTCPCheck will tell the service provider to check the service address -// and port every `t` interval. It will enabled only if `t` is greater than 0. -// This option is for registry using Consul, see `TCP + Interval` more -// information [1]. -// -// [1] https://www.consul.io/docs/agent/checks.html -// -func RegisterTCPCheck(t time.Duration) RegisterOption { - return func(o *RegisterOptions) { - if t > time.Duration(0) { - o.TCPCheck = true - o.Interval = t - } - } -} - // Watch a service func WatchService(name string) WatchOption { return func(o *WatchOptions) { diff --git a/server/options.go b/server/options.go index 5fd4e4bb..5b9aa98e 100644 --- a/server/options.go +++ b/server/options.go @@ -25,9 +25,7 @@ type Options struct { HdlrWrappers []HandlerWrapper SubWrappers []SubscriberWrapper - RegisterTCPCheck bool - RegisterTTL time.Duration - RegisterInterval time.Duration + RegisterTTL time.Duration // Debug Handler which can be set by a user DebugHandler debug.DebugHandler @@ -166,23 +164,6 @@ func RegisterTTL(t time.Duration) Option { } } -// -// RegisterTCPCheck will tell the service provider to check the service address -// and port every `t` interval. It will enabled only if `t` is greater than 0. -// This option is for registry using Consul, see `TCP + Interval` more -// information [1]. -// -// [1] https://www.consul.io/docs/agent/checks.html -// -func RegisterTCPCheck(t time.Duration) Option { - return func(o *Options) { - if t > time.Duration(0) { - o.RegisterTCPCheck = true - o.RegisterInterval = t - } - } -} - // Wait tells the server to wait for requests to finish before exiting func Wait(b bool) Option { return func(o *Options) { diff --git a/server/rpc_server.go b/server/rpc_server.go index f1ec257b..aa1285d9 100644 --- a/server/rpc_server.go +++ b/server/rpc_server.go @@ -278,10 +278,7 @@ func (s *rpcServer) Register() error { } // create registry options - rOpts := []registry.RegisterOption{ - registry.RegisterTTL(config.RegisterTTL), - registry.RegisterTCPCheck(config.RegisterInterval), - } + rOpts := []registry.RegisterOption{registry.RegisterTTL(config.RegisterTTL)} if err := config.Registry.Register(service, rOpts...); err != nil { return err From 1eb4398b6c028c47aded4896dcaf806ca66c346f Mon Sep 17 00:00:00 2001 From: Shulhan Date: Wed, 21 Mar 2018 18:17:56 +0700 Subject: [PATCH 4/6] registry/consul: rename "RegisterTCPCheck" to "TCPCheck" --- registry/consul/options.go | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/registry/consul/options.go b/registry/consul/options.go index 8957657c..09329874 100644 --- a/registry/consul/options.go +++ b/registry/consul/options.go @@ -18,13 +18,13 @@ func Config(c *consul.Config) registry.Option { } // -// RegisterTCPCheck will tell the service provider to check the service address +// TCPCheck will tell the service provider to check the service address // and port every `t` interval. It will enabled only if `t` is greater than 0. // See `TCP + Interval` for more information [1]. // // [1] https://www.consul.io/docs/agent/checks.html // -func RegisterTCPCheck(t time.Duration) registry.Option { +func TCPCheck(t time.Duration) registry.Option { return func(o *registry.Options) { if t <= time.Duration(0) { return @@ -32,7 +32,6 @@ func RegisterTCPCheck(t time.Duration) registry.Option { if o.Context == nil { o.Context = context.Background() } - o.Context = context.WithValue(o.Context, - registry.ConsulRegisterTCPCheckKey, t) + o.Context = context.WithValue(o.Context, "consul_register_tcp_check", t) } } From 65a90f5a219683c6bd19045cefc2064056ff77c3 Mon Sep 17 00:00:00 2001 From: Shulhan Date: Wed, 21 Mar 2018 18:18:48 +0700 Subject: [PATCH 5/6] registry.Register: use local variable to get context value --- registry/consul_registry.go | 21 +++++++++------------ registry/options.go | 4 +--- 2 files changed, 10 insertions(+), 15 deletions(-) diff --git a/registry/consul_registry.go b/registry/consul_registry.go index dda716a4..514e29b1 100644 --- a/registry/consul_registry.go +++ b/registry/consul_registry.go @@ -14,10 +14,6 @@ import ( hash "github.com/mitchellh/hashstructure" ) -const ( - ConsulRegisterTCPCheckKey = "consul_register_tcp_check" -) - type consulRegistry struct { Address string Client *consul.Client @@ -137,17 +133,18 @@ func (c *consulRegistry) Register(s *Service, opts ...RegisterOption) error { return errors.New("Require at least one node") } + var regTCPCheck bool + var regInterval time.Duration + var options RegisterOptions for _, o := range opts { o(&options) } if c.opts.Context != nil { - tcpCheckInterval, ok := c.opts.Context. - Value(ConsulRegisterTCPCheckKey).(time.Duration) - if ok { - options.TCPCheck = true - options.Interval = tcpCheckInterval + if tcpCheckInterval, ok := c.opts.Context.Value("consul_register_tcp_check").(time.Duration); ok { + regTCPCheck = true + regInterval = tcpCheckInterval } } @@ -181,12 +178,12 @@ func (c *consulRegistry) Register(s *Service, opts ...RegisterOption) error { var check *consul.AgentServiceCheck - if options.TCPCheck { - deregTTL := getDeregisterTTL(options.Interval) + if regTCPCheck { + deregTTL := getDeregisterTTL(regInterval) check = &consul.AgentServiceCheck{ TCP: fmt.Sprintf("%s:%d", node.Address, node.Port), - Interval: fmt.Sprintf("%v", options.Interval), + Interval: fmt.Sprintf("%v", regInterval), DeregisterCriticalServiceAfter: fmt.Sprintf("%v", deregTTL), } diff --git a/registry/options.go b/registry/options.go index cb687834..aa9c3f4f 100644 --- a/registry/options.go +++ b/registry/options.go @@ -18,9 +18,7 @@ type Options struct { } type RegisterOptions struct { - TCPCheck bool - TTL time.Duration - Interval time.Duration + TTL time.Duration // Other options for implementations of the interface // can be stored in a context Context context.Context From 44b934d458be56f57d24f9b761b7201a887b678f Mon Sep 17 00:00:00 2001 From: Shulhan Date: Wed, 21 Mar 2018 21:57:04 +0700 Subject: [PATCH 6/6] registry: rename context key "consul_register_tcp_check" to "consul_tcp_check" --- registry/consul/options.go | 2 +- registry/consul_registry.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/registry/consul/options.go b/registry/consul/options.go index 09329874..3feb9cef 100644 --- a/registry/consul/options.go +++ b/registry/consul/options.go @@ -32,6 +32,6 @@ func TCPCheck(t time.Duration) registry.Option { if o.Context == nil { o.Context = context.Background() } - o.Context = context.WithValue(o.Context, "consul_register_tcp_check", t) + o.Context = context.WithValue(o.Context, "consul_tcp_check", t) } } diff --git a/registry/consul_registry.go b/registry/consul_registry.go index 514e29b1..a58da58e 100644 --- a/registry/consul_registry.go +++ b/registry/consul_registry.go @@ -142,7 +142,7 @@ func (c *consulRegistry) Register(s *Service, opts ...RegisterOption) error { } if c.opts.Context != nil { - if tcpCheckInterval, ok := c.opts.Context.Value("consul_register_tcp_check").(time.Duration); ok { + if tcpCheckInterval, ok := c.opts.Context.Value("consul_tcp_check").(time.Duration); ok { regTCPCheck = true regInterval = tcpCheckInterval }