Ignore "no such process" error (#1686)

* Cleanup how status is updated for service. Ignore "no such process" error as it could be that the pid died

* add nice error log to record process error exit

runtime/default.go

@@ -373,7 +373,6 @@ func (r *runtime) Create(s *Service, opts ...CreateOption) error {
 	if err := service.Start(); err != nil {
 		return err
 	}
-
 	// save service
 	r.namespaces[options.Namespace][serviceKey(s)] = service
 
@@ -536,7 +535,7 @@ func (r *runtime) Read(opts ...ReadOption) ([]*Service, error) {
 	return services, nil
 }
 
-// Update attemps to update the service
+// Update attempts to update the service
 func (r *runtime) Update(s *Service, opts ...UpdateOption) error {
 	var options UpdateOptions
 	for _, o := range opts {
@@ -565,7 +564,8 @@ func (r *runtime) Update(s *Service, opts ...UpdateOption) error {
 		return errors.New("Service not found")
 	}
 
-	if err := service.Stop(); err != nil {
+	if err := service.Stop(); err != nil && err.Error() != "no such process" {
+		logger.Errorf("Error stopping service %s: %s", service.Name, err)
 		return err
 	}
 

runtime/kubernetes/kubernetes.go

@@ -167,11 +167,10 @@ func (k *kubernetes) getService(labels map[string]string, opts ...client.GetOpti
 
 			// parse out deployment status and inject into service metadata
 			if len(kdep.Status.Conditions) > 0 {
-				svc.Metadata["status"] = kdep.Status.Conditions[0].Type
+				svc.Status(kdep.Status.Conditions[0].Type, nil)
 				svc.Metadata["started"] = kdep.Status.Conditions[0].LastUpdateTime
-				delete(svc.Metadata, "error")
 			} else {
-				svc.Metadata["status"] = "n/a"
+				svc.Status("n/a", nil)
 			}
 
 			// get the real status
@@ -214,8 +213,7 @@ func (k *kubernetes) getService(labels map[string]string, opts ...client.GetOpti
 					}
 				}
 				// TODO: set from terminated
-
-				svc.Metadata["status"] = status
+				svc.Status(status, nil)
 			}
 
 			// save deployment

runtime/kubernetes/service.go

@@ -3,6 +3,7 @@ package kubernetes
 import (
 	"encoding/json"
 	"strings"
+	"time"
 
 	"github.com/micro/go-micro/v2/logger"
 	"github.com/micro/go-micro/v2/runtime"
@@ -185,8 +186,10 @@ func (s *service) Update(k client.Client, opts ...client.UpdateOption) error {
 }
 
 func (s *service) Status(status string, err error) {
+	s.Metadata["lastStatusUpdate"] = time.Now().Format(time.RFC3339)
 	if err == nil {
 		s.Metadata["status"] = status
+		delete(s.Metadata, "error")
 		return
 	}
 	s.Metadata["status"] = "error"

runtime/service.go

@@ -93,7 +93,7 @@ func (s *service) Running() bool {
 	return s.running
 }
 
-// Start stars the service
+// Start starts the service
 func (s *service) Start() error {
 	s.Lock()
 	defer s.Unlock()
@@ -110,10 +110,7 @@ func (s *service) Start() error {
 	if s.Metadata == nil {
 		s.Metadata = make(map[string]string)
 	}
-
-	s.Metadata["status"] = "starting"
-	// delete any existing error
-	delete(s.Metadata, "error")
+	s.Status("starting", nil)
 
 	// TODO: pull source & build binary
 	if logger.V(logger.DebugLevel, logger.DefaultLogger) {
@@ -122,17 +119,15 @@ func (s *service) Start() error {
 
 	p, err := s.Process.Fork(s.Exec)
 	if err != nil {
-		s.Metadata["status"] = "error"
-		s.Metadata["error"] = err.Error()
+		s.Status("error", err)
 		return err
 	}
-
 	// set the pid
 	s.PID = p
 	// set to running
 	s.running = true
 	// set status
-	s.Metadata["status"] = "running"
+	s.Status("running", nil)
 	// set started
 	s.Metadata["started"] = time.Now().Format(time.RFC3339)
 
@@ -146,6 +141,18 @@ func (s *service) Start() error {
 	return nil
 }
 
+// Status updates the status of the service. Assumes it's called under a lock as it mutates state
+func (s *service) Status(status string, err error) {
+	s.Metadata["lastStatusUpdate"] = time.Now().Format(time.RFC3339)
+	s.Metadata["status"] = status
+	if err == nil {
+		delete(s.Metadata, "error")
+		return
+	}
+	s.Metadata["error"] = err.Error()
+
+}
+
 // Stop stops the service
 func (s *service) Stop() error {
 	s.Lock()
@@ -163,18 +170,17 @@ func (s *service) Stop() error {
 		}
 
 		// set status
-		s.Metadata["status"] = "stopping"
+		s.Status("stopping", nil)
 
 		// kill the process
 		err := s.Process.Kill(s.PID)
-		if err != nil {
-			return err
+		if err == nil {
+			// wait for it to exit
+			s.Process.Wait(s.PID)
 		}
-		// wait for it to exit
-		s.Process.Wait(s.PID)
 
 		// set status
-		s.Metadata["status"] = "stopped"
+		s.Status("stopped", err)
 
 		// return the kill error
 		return err
@@ -198,14 +204,16 @@ func (s *service) Wait() {
 
 	// save the error
 	if err != nil {
+		if logger.V(logger.ErrorLevel, logger.DefaultLogger) {
+			logger.Errorf("Service %s terminated with error %s", s.Name, err)
+		}
 		s.retries++
-		s.Metadata["status"] = "error"
-		s.Metadata["error"] = err.Error()
+		s.Status("error", err)
 		s.Metadata["retries"] = strconv.Itoa(s.retries)
 
 		s.err = err
 	} else {
-		s.Metadata["status"] = "done"
+		s.Status("done", nil)
 	}
 
 	// no longer running