Registry router fixes (#1961)

* only cache routes if told to do so

* Use roundrobin selector and retry in proxy

* Update lookup to require service

* Fix compile

* Fix compile

* Update

* Update

* rename query to lookup

* Update router.go

* Update

registry.go

@@ -47,7 +47,7 @@ func NewRouter(opts ...router.Option) router.Router {
 
 	// create the new table, passing the fetchRoute method in as a fallback if
 	// the table doesn't contain the result for a query.
-	r.table = newTable(r.lookup)
+	r.table = newTable()
 
 	// start the router
 	r.start()
@@ -241,8 +241,41 @@ func (r *rtr) loadRoutes(reg registry.Registry) error {
 	return nil
 }
 
+// Close the router
+func (r *rtr) Close() error {
+	r.Lock()
+	defer r.Unlock()
+
+	select {
+	case <-r.exit:
+		return nil
+	default:
+		if !r.running {
+			return nil
+		}
+		close(r.exit)
+
+	}
+
+	r.running = false
+	return nil
+}
+
 // lookup retrieves all the routes for a given service and creates them in the routing table
-func (r *rtr) lookup(service string) ([]router.Route, error) {
+func (r *rtr) Lookup(service string, opts ...router.LookupOption) ([]router.Route, error) {
+	q := router.NewLookup(opts...)
+
+	// if we find the routes filter and return them
+	routes, err := r.table.Query(service)
+	if err == nil {
+		routes = router.Filter(routes, q)
+		if len(routes) == 0 {
+			return nil, router.ErrRouteNotFound
+		}
+		return routes, nil
+	}
+
+	// lookup the route
 	logger.Tracef("Fetching route for %s domain: %v", service, registry.WildcardDomain)
 
 	services, err := r.options.Registry.GetService(service, registry.GetDomain(registry.WildcardDomain))
@@ -254,8 +287,6 @@ func (r *rtr) lookup(service string) ([]router.Route, error) {
 		return nil, fmt.Errorf("failed getting services: %v", err)
 	}
 
-	var routes []router.Route
-
 	for _, srv := range services {
 		domain := getDomain(srv)
 		// TODO: should we continue to send the event indicating we created a route?
@@ -263,6 +294,17 @@ func (r *rtr) lookup(service string) ([]router.Route, error) {
 		routes = append(routes, r.createRoutes(srv, domain)...)
 	}
 
+	// if we're supposed to cache then save the routes
+	if r.options.Cache {
+		for _, route := range routes {
+			r.table.Create(route)
+		}
+	}
+
+	routes = router.Filter(routes, q)
+	if len(routes) == 0 {
+		return nil, router.ErrRouteNotFound
+	}
 	return routes, nil
 }
 
@@ -324,13 +366,6 @@ func (r *rtr) start() error {
 		return nil
 	}
 
-	if r.options.Precache {
-		// add all local service routes into the routing table
-		if err := r.loadRoutes(r.options.Registry); err != nil {
-			return fmt.Errorf("failed loading registry routes: %s", err)
-		}
-	}
-
 	// add default gateway into routing table
 	if r.options.Gateway != "" {
 		// note, the only non-default value is the gateway
@@ -350,25 +385,59 @@ func (r *rtr) start() error {
 
 	// create error and exit channels
 	r.exit = make(chan bool)
+	r.running = true
 
-	// periodically refresh all the routes
+	// only cache if told to do so
+	if !r.options.Cache {
+		return nil
+	}
+
+	// create a refresh notify channel
+	refresh := make(chan bool, 1)
+
+	// fires the refresh for loading routes
+	refreshRoutes := func() {
+		select {
+		case refresh <- true:
+		default:
+		}
+	}
+
+	// refresh all the routes in the event of a failure watching the registry
 	go func() {
-		t1 := time.NewTicker(RefreshInterval)
-		defer t1.Stop()
+		var lastRefresh time.Time
 
-		t2 := time.NewTicker(PruneInterval)
-		defer t2.Stop()
+		// load a refresh
+		refreshRoutes()
 
 		for {
 			select {
 			case <-r.exit:
 				return
-			case <-t2.C:
-				r.table.pruneRoutes(RefreshInterval)
-			case <-t1.C:
+			case <-refresh:
+				// don't refresh if we've done so in the past minute
+				if !lastRefresh.IsZero() && time.Since(lastRefresh) < time.Minute {
+					continue
+				}
+
+				// load new routes
 				if err := r.loadRoutes(r.options.Registry); err != nil {
 					logger.Debugf("failed refreshing registry routes: %s", err)
+					// in this don't prune
+					continue
 				}
+
+				// first time so nothing to prune
+				if !lastRefresh.IsZero() {
+					// prune any routes since last refresh since we've
+					// updated basically everything we care about
+					r.table.pruneRoutes(time.Since(lastRefresh))
+				}
+
+				// update the refresh time
+				lastRefresh = time.Now()
+			case <-time.After(RefreshInterval):
+				refreshRoutes()
 			}
 		}
 	}()
@@ -386,6 +455,8 @@ func (r *rtr) start() error {
 						logger.Debugf("failed creating registry watcher: %v", err)
 					}
 					time.Sleep(time.Second)
+					// in the event of an error reload routes
+					refreshRoutes()
 					continue
 				}
 
@@ -395,46 +466,21 @@ func (r *rtr) start() error {
 						logger.Debugf("Error watching the registry: %v", err)
 					}
 					time.Sleep(time.Second)
+					// in the event of an error reload routes
+					refreshRoutes()
 				}
 			}
 		}
 	}()
 
-	r.running = true
-
 	return nil
 }
 
-// Lookup routes in the routing table
-func (r *rtr) Lookup(q ...router.QueryOption) ([]router.Route, error) {
-	return r.Table().Query(q...)
-}
-
 // Watch routes
 func (r *rtr) Watch(opts ...router.WatchOption) (router.Watcher, error) {
 	return r.table.Watch(opts...)
 }
 
-// Close the router
-func (r *rtr) Close() error {
-	r.Lock()
-	defer r.Unlock()
-
-	select {
-	case <-r.exit:
-		return nil
-	default:
-		if !r.running {
-			return nil
-		}
-		close(r.exit)
-
-	}
-
-	r.running = false
-	return nil
-}
-
 // String prints debugging information about router
 func (r *rtr) String() string {
 	return "registry"