From 47bdd5c99383c41776d2c4499ee0ef9960d81a6d Mon Sep 17 00:00:00 2001 From: Dominic Wong Date: Mon, 8 Jun 2020 10:47:25 +0100 Subject: [PATCH] Ignore "no such process" error (#1686) * Cleanup how status is updated for service. Ignore "no such process" error as it could be that the pid died * add nice error log to record process error exit --- runtime/default.go | 6 ++--- runtime/kubernetes/kubernetes.go | 8 +++--- runtime/kubernetes/service.go | 3 +++ runtime/service.go | 44 +++++++++++++++++++------------- 4 files changed, 35 insertions(+), 26 deletions(-) diff --git a/runtime/default.go b/runtime/default.go index 61a1f562..28ecc4b8 100644 --- a/runtime/default.go +++ b/runtime/default.go @@ -373,7 +373,6 @@ func (r *runtime) Create(s *Service, opts ...CreateOption) error { if err := service.Start(); err != nil { return err } - // save service r.namespaces[options.Namespace][serviceKey(s)] = service @@ -536,7 +535,7 @@ func (r *runtime) Read(opts ...ReadOption) ([]*Service, error) { return services, nil } -// Update attemps to update the service +// Update attempts to update the service func (r *runtime) Update(s *Service, opts ...UpdateOption) error { var options UpdateOptions for _, o := range opts { @@ -565,7 +564,8 @@ func (r *runtime) Update(s *Service, opts ...UpdateOption) error { return errors.New("Service not found") } - if err := service.Stop(); err != nil { + if err := service.Stop(); err != nil && err.Error() != "no such process" { + logger.Errorf("Error stopping service %s: %s", service.Name, err) return err } diff --git a/runtime/kubernetes/kubernetes.go b/runtime/kubernetes/kubernetes.go index 5ce110c6..c464d381 100644 --- a/runtime/kubernetes/kubernetes.go +++ b/runtime/kubernetes/kubernetes.go @@ -167,11 +167,10 @@ func (k *kubernetes) getService(labels map[string]string, opts ...client.GetOpti // parse out deployment status and inject into service metadata if len(kdep.Status.Conditions) > 0 { - svc.Metadata["status"] = kdep.Status.Conditions[0].Type + svc.Status(kdep.Status.Conditions[0].Type, nil) svc.Metadata["started"] = kdep.Status.Conditions[0].LastUpdateTime - delete(svc.Metadata, "error") } else { - svc.Metadata["status"] = "n/a" + svc.Status("n/a", nil) } // get the real status @@ -214,8 +213,7 @@ func (k *kubernetes) getService(labels map[string]string, opts ...client.GetOpti } } // TODO: set from terminated - - svc.Metadata["status"] = status + svc.Status(status, nil) } // save deployment diff --git a/runtime/kubernetes/service.go b/runtime/kubernetes/service.go index b3c02395..4cb71e37 100644 --- a/runtime/kubernetes/service.go +++ b/runtime/kubernetes/service.go @@ -3,6 +3,7 @@ package kubernetes import ( "encoding/json" "strings" + "time" "github.com/micro/go-micro/v2/logger" "github.com/micro/go-micro/v2/runtime" @@ -185,8 +186,10 @@ func (s *service) Update(k client.Client, opts ...client.UpdateOption) error { } func (s *service) Status(status string, err error) { + s.Metadata["lastStatusUpdate"] = time.Now().Format(time.RFC3339) if err == nil { s.Metadata["status"] = status + delete(s.Metadata, "error") return } s.Metadata["status"] = "error" diff --git a/runtime/service.go b/runtime/service.go index 2733b01e..5c6ba621 100644 --- a/runtime/service.go +++ b/runtime/service.go @@ -93,7 +93,7 @@ func (s *service) Running() bool { return s.running } -// Start stars the service +// Start starts the service func (s *service) Start() error { s.Lock() defer s.Unlock() @@ -110,10 +110,7 @@ func (s *service) Start() error { if s.Metadata == nil { s.Metadata = make(map[string]string) } - - s.Metadata["status"] = "starting" - // delete any existing error - delete(s.Metadata, "error") + s.Status("starting", nil) // TODO: pull source & build binary if logger.V(logger.DebugLevel, logger.DefaultLogger) { @@ -122,17 +119,15 @@ func (s *service) Start() error { p, err := s.Process.Fork(s.Exec) if err != nil { - s.Metadata["status"] = "error" - s.Metadata["error"] = err.Error() + s.Status("error", err) return err } - // set the pid s.PID = p // set to running s.running = true // set status - s.Metadata["status"] = "running" + s.Status("running", nil) // set started s.Metadata["started"] = time.Now().Format(time.RFC3339) @@ -146,6 +141,18 @@ func (s *service) Start() error { return nil } +// Status updates the status of the service. Assumes it's called under a lock as it mutates state +func (s *service) Status(status string, err error) { + s.Metadata["lastStatusUpdate"] = time.Now().Format(time.RFC3339) + s.Metadata["status"] = status + if err == nil { + delete(s.Metadata, "error") + return + } + s.Metadata["error"] = err.Error() + +} + // Stop stops the service func (s *service) Stop() error { s.Lock() @@ -163,18 +170,17 @@ func (s *service) Stop() error { } // set status - s.Metadata["status"] = "stopping" + s.Status("stopping", nil) // kill the process err := s.Process.Kill(s.PID) - if err != nil { - return err + if err == nil { + // wait for it to exit + s.Process.Wait(s.PID) } - // wait for it to exit - s.Process.Wait(s.PID) // set status - s.Metadata["status"] = "stopped" + s.Status("stopped", err) // return the kill error return err @@ -198,14 +204,16 @@ func (s *service) Wait() { // save the error if err != nil { + if logger.V(logger.ErrorLevel, logger.DefaultLogger) { + logger.Errorf("Service %s terminated with error %s", s.Name, err) + } s.retries++ - s.Metadata["status"] = "error" - s.Metadata["error"] = err.Error() + s.Status("error", err) s.Metadata["retries"] = strconv.Itoa(s.retries) s.err = err } else { - s.Metadata["status"] = "done" + s.Status("done", nil) } // no longer running