api/router/registry: extract path based parameters from url to req (#1530)

* api/router/registry: extract path based parameters from url to req
* api/handler/rpc: fix empty body request parsing
* bundle grpc-gateway util funcs

Signed-off-by: Vasiliy Tolstov <v.tolstov@unistack.org>

registry.go

@@ -6,16 +6,25 @@ import (
 	"fmt"
 	"net/http"
 	"regexp"
+	"strings"
 	"sync"
 	"time"
 
 	"github.com/micro/go-micro/v2/api"
 	"github.com/micro/go-micro/v2/api/router"
+	"github.com/micro/go-micro/v2/api/router/util"
 	"github.com/micro/go-micro/v2/logger"
+	"github.com/micro/go-micro/v2/metadata"
 	"github.com/micro/go-micro/v2/registry"
 	"github.com/micro/go-micro/v2/registry/cache"
 )
 
+// endpoint struct, that holds compiled pcre
+type endpoint struct {
+	hostregs []*regexp.Regexp
+	pathregs []util.Pattern
+}
+
 // router is the default router
 type registryRouter struct {
 	exit chan bool
@@ -26,6 +35,8 @@ type registryRouter struct {
 
 	sync.RWMutex
 	eps map[string]*api.Service
+	// compiled regexp for host and path
+	ceps map[string]*endpoint
 }
 
 func (r *registryRouter) isClosed() bool {
@@ -67,6 +78,7 @@ func (r *registryRouter) refresh() {
 		}
 
 		// refresh list in 10 minutes... cruft
+		// use registry watching
 		select {
 		case <-time.After(time.Minute * 10):
 		case <-r.exit:
@@ -109,11 +121,11 @@ func (r *registryRouter) store(services []*registry.Service) {
 		names[service.Name] = true
 
 		// map per endpoint
-		for _, endpoint := range service.Endpoints {
+		for _, sep := range service.Endpoints {
 			// create a key service:endpoint_name
-			key := fmt.Sprintf("%s:%s", service.Name, endpoint.Name)
+			key := fmt.Sprintf("%s.%s", service.Name, sep.Name)
 			// decode endpoint
-			end := api.Decode(endpoint.Metadata)
+			end := api.Decode(sep.Metadata)
 
 			// if we got nothing skip
 			if err := api.Validate(end); err != nil {
@@ -154,8 +166,44 @@ func (r *registryRouter) store(services []*registry.Service) {
 	}
 
 	// now set the eps we have
-	for name, endpoint := range eps {
+	for name, ep := range eps {
-		r.eps[name] = endpoint
+		r.eps[name] = ep
+		cep := &endpoint{}
+
+		for _, h := range ep.Endpoint.Host {
+			if h == "" || h == "*" {
+				continue
+			}
+			hostreg, err := regexp.CompilePOSIX(h)
+			if err != nil {
+				if logger.V(logger.TraceLevel, logger.DefaultLogger) {
+					logger.Tracef("endpoint have invalid host regexp: %v", err)
+				}
+				continue
+			}
+			cep.hostregs = append(cep.hostregs, hostreg)
+		}
+
+		for _, p := range ep.Endpoint.Path {
+			rule, err := util.Parse(p)
+			if err != nil {
+				if logger.V(logger.TraceLevel, logger.DefaultLogger) {
+					logger.Tracef("endpoint have invalid path pattern: %v", err)
+				}
+				continue
+			}
+			tpl := rule.Compile()
+			pathreg, err := util.NewPattern(tpl.Version, tpl.OpCodes, tpl.Pool, "")
+			if err != nil {
+				if logger.V(logger.TraceLevel, logger.DefaultLogger) {
+					logger.Tracef("endpoint have invalid path pattern: %v", err)
+				}
+				continue
+			}
+			cep.pathregs = append(cep.pathregs, pathreg)
+		}
+
+		r.ceps[name] = cep
 	}
 }
 
@@ -239,60 +287,89 @@ func (r *registryRouter) Endpoint(req *http.Request) (*api.Service, error) {
 	r.RLock()
 	defer r.RUnlock()
 
+	var idx int
+	if len(req.URL.Path) > 0 && req.URL.Path != "/" {
+		idx = 1
+	}
+	path := strings.Split(req.URL.Path[idx:], "/")
+
 	// use the first match
 	// TODO: weighted matching
-	for _, e := range r.eps {
+	for n, e := range r.eps {
+		cep, ok := r.ceps[n]
+		if !ok {
+			continue
+		}
 		ep := e.Endpoint
-
+		var mMatch, hMatch, pMatch bool
-		// match
+		// 1. try method
-		var pathMatch, hostMatch, methodMatch bool
+	methodLoop:
-
-		// 1. try method GET, POST, PUT, etc
-		// 2. try host example.com, foobar.com, etc
-		// 3. try path /foo/bar, /bar/baz, etc
-
-		// 1. try match method
 		for _, m := range ep.Method {
-			if req.Method == m {
+			if m == req.Method {
-				methodMatch = true
+				mMatch = true
-				break
+				break methodLoop
 			}
 		}
-
+		if !mMatch {
-		// no match on method pass
-		if len(ep.Method) > 0 && !methodMatch {
 			continue
 		}
-
+		if logger.V(logger.DebugLevel, logger.DefaultLogger) {
-		// 2. try match host
+			logger.Debugf("api method match %s", req.Method)
-		for _, h := range ep.Host {
-			if req.Host == h {
-				hostMatch = true
-				break
-			}
 		}
 
-		// no match on host pass
+		// 2. try host
-		if len(ep.Host) > 0 && !hostMatch {
+		if len(ep.Host) == 0 {
+			hMatch = true
+		} else {
+		hostLoop:
+			for idx, h := range ep.Host {
+				if h == "" || h == "*" {
+					hMatch = true
+					break hostLoop
+				} else {
+					if cep.hostregs[idx].MatchString(req.URL.Host) {
+						hMatch = true
+						break hostLoop
+					}
+				}
+			}
+		}
+		if !hMatch {
 			continue
 		}
-
+		if logger.V(logger.DebugLevel, logger.DefaultLogger) {
-		// 3. try match paths
+			logger.Debugf("api host match %s", req.URL.Host)
-		for _, p := range ep.Path {
-			re, err := regexp.CompilePOSIX(p)
-			if err == nil && re.MatchString(req.URL.Path) {
-				pathMatch = true
-				break
-			}
 		}
 
-		// no match pass
+		// 3. try path
-		if len(ep.Path) > 0 && !pathMatch {
+		// 3. try path
+	pathLoop:
+		for _, pathreg := range cep.pathregs {
+			matches, err := pathreg.Match(path, "")
+			if err != nil {
+				if logger.V(logger.DebugLevel, logger.DefaultLogger) {
+					logger.Debugf("api path not match %s != %v", path, pathreg)
+				}
+				continue
+			}
+			pMatch = true
+			ctx := req.Context()
+			md, ok := metadata.FromContext(ctx)
+			if !ok {
+				md = make(metadata.Metadata)
+			}
+			for k, v := range matches {
+				md[fmt.Sprintf("x-api-field-%s", k)] = v
+			}
+			md["x-api-body"] = ep.Body
+			*req = *req.Clone(metadata.NewContext(ctx, md))
+			break pathLoop
+		}
+		if !pMatch {
 			continue
 		}
 
 		// TODO: Percentage traffic
-
 		// we got here, so its a match
 		return e, nil
 	}
@@ -377,6 +454,7 @@ func newRouter(opts ...router.Option) *registryRouter {
 		opts: options,
 		rc:   cache.New(options.Registry),
 		eps:  make(map[string]*api.Service),
+		ceps: make(map[string]*endpoint),
 	}
 	go r.watch()
 	go r.refresh()

registry_test.go

@@ -1,136 +0,0 @@
-package registry
-
-import (
-	"fmt"
-	"net/http"
-	"net/url"
-	"testing"
-
-	"github.com/micro/go-micro/v2/api"
-)
-
-func TestRouter(t *testing.T) {
-	r := newRouter()
-
-	compare := func(expect, got []string) bool {
-		// no data to compare, return true
-		if len(expect) == 0 && len(got) == 0 {
-			return true
-		}
-		// no data expected but got some return false
-		if len(expect) == 0 && len(got) > 0 {
-			return false
-		}
-
-		// compare expected with what we got
-		for _, e := range expect {
-			var seen bool
-			for _, g := range got {
-				if e == g {
-					seen = true
-					break
-				}
-			}
-			if !seen {
-				return false
-			}
-		}
-
-		// we're done, return true
-		return true
-	}
-
-	testData := []struct {
-		e *api.Endpoint
-		r *http.Request
-		m bool
-	}{
-		{
-			e: &api.Endpoint{
-				Name:   "Foo.Bar",
-				Host:   []string{"example.com"},
-				Method: []string{"GET"},
-				Path:   []string{"/foo"},
-			},
-			r: &http.Request{
-				Host:   "example.com",
-				Method: "GET",
-				URL: &url.URL{
-					Path: "/foo",
-				},
-			},
-			m: true,
-		},
-		{
-			e: &api.Endpoint{
-				Name:   "Bar.Baz",
-				Host:   []string{"example.com", "foo.com"},
-				Method: []string{"GET", "POST"},
-				Path:   []string{"/foo/bar"},
-			},
-			r: &http.Request{
-				Host:   "foo.com",
-				Method: "POST",
-				URL: &url.URL{
-					Path: "/foo/bar",
-				},
-			},
-			m: true,
-		},
-		{
-			e: &api.Endpoint{
-				Name:   "Test.Cruft",
-				Host:   []string{"example.com", "foo.com"},
-				Method: []string{"GET", "POST"},
-				Path:   []string{"/xyz"},
-			},
-			r: &http.Request{
-				Host:   "fail.com",
-				Method: "DELETE",
-				URL: &url.URL{
-					Path: "/test/fail",
-				},
-			},
-			m: false,
-		},
-	}
-
-	for _, d := range testData {
-		key := fmt.Sprintf("%s:%s", "test.service", d.e.Name)
-		r.eps[key] = &api.Service{
-			Endpoint: d.e,
-		}
-	}
-
-	for _, d := range testData {
-		e, err := r.Endpoint(d.r)
-		if d.m && err != nil {
-			t.Fatalf("expected match, got %v", err)
-		}
-		if !d.m && err == nil {
-			t.Fatal("expected error got match")
-		}
-		// skip testing the non match
-		if !d.m {
-			continue
-		}
-
-		ep := e.Endpoint
-
-		// test the match
-		if d.e.Name != ep.Name {
-			t.Fatalf("expected %v got %v", d.e.Name, ep.Name)
-		}
-		if ok := compare(d.e.Method, ep.Method); !ok {
-			t.Fatalf("expected %v got %v", d.e.Method, ep.Method)
-		}
-		if ok := compare(d.e.Path, ep.Path); !ok {
-			t.Fatalf("expected %v got %v", d.e.Path, ep.Path)
-		}
-		if ok := compare(d.e.Host, ep.Host); !ok {
-			t.Fatalf("expected %v got %v", d.e.Host, ep.Host)
-		}
-
-	}
-
-}