Preserve cache in the face of failure

registry/cache/rcache.go

@@ -36,7 +36,13 @@ type cache struct {
 	ttls    map[string]time.Time
 	watched map[string]bool
 
+	// used to stop the cache
 	exit chan bool
+
+	// status of the registry
+	// used to hold onto the cache
+	// in failure state
+	status error
 }
 
 var (
@@ -50,6 +56,18 @@ func backoff(attempts int) time.Duration {
 	return time.Duration(math.Pow(10, float64(attempts))) * time.Millisecond
 }
 
+func (c *cache) getStatus() error {
+	c.RLock()
+	defer c.RUnlock()
+	return c.status
+}
+
+func (c *cache) setStatus(err error) {
+	c.Lock()
+	c.status = err
+	c.Unlock()
+}
+
 // isValid checks if the service is valid
 func (c *cache) isValid(services []*registry.Service, ttl time.Time) bool {
 	// no services exist
@@ -81,6 +99,11 @@ func (c *cache) quit() bool {
 }
 
 func (c *cache) del(service string) {
+	// don't blow away cache in error state
+	if err := c.getStatus(); err != nil {
+		return
+	}
+	// otherwise delete entries
 	delete(c.cache, service)
 	delete(c.ttls, service)
 }
@@ -105,13 +128,26 @@ func (c *cache) get(service string) ([]*registry.Service, error) {
 	}
 
 	// get does the actual request for a service and cache it
-	get := func(service string) ([]*registry.Service, error) {
+	get := func(service string, cached []*registry.Service) ([]*registry.Service, error) {
 		// ask the registry
 		services, err := c.Registry.GetService(service)
 		if err != nil {
+			// check the cache
+			if len(cached) > 0 {
+				// set the error status
+				c.setStatus(err)
+				// return the stale cache
+				return registry.Copy(cached), nil
+			}
+			// otherwise return error
 			return nil, err
 		}
 
+		// reset the status
+		if c.getStatus(); err != nil {
+			c.setStatus(nil)
+		}
+
 		// cache results
 		c.Lock()
 		c.set(service, registry.Copy(services))
@@ -129,7 +165,7 @@ func (c *cache) get(service string) ([]*registry.Service, error) {
 	c.RUnlock()
 
 	// get and return services
-	return get(service)
+	return get(service, services)
 }
 
 func (c *cache) set(service string, services []*registry.Service) {
@@ -283,6 +319,7 @@ func (c *cache) run(service string) {
 			}
 
 			d := backoff(a)
+			c.setStatus(err)
 
 			if a > 3 {
 				log.Log("rcache: ", err, " backing off ", d)
@@ -305,6 +342,7 @@ func (c *cache) run(service string) {
 			}
 
 			d := backoff(b)
+			c.setStatus(err)
 
 			if b > 3 {
 				log.Log("rcache: ", err, " backing off ", d)
@@ -348,6 +386,13 @@ func (c *cache) watch(w registry.Watcher) error {
 			close(stop)
 			return err
 		}
+
+		// reset the error status since we succeeded
+		if err := c.getStatus(); err != nil {
+			// reset status
+			c.setStatus(nil)
+		}
+
 		c.update(res)
 	}
 }