Merge pull request #63 from micro/encoding

Fix Consul WARN issues for tags

registry/consul_registry.go

@@ -2,7 +2,6 @@ package registry
 
 import (
 	"crypto/tls"
-	"encoding/json"
 	"errors"
 	"fmt"
 	"net"
@@ -41,74 +40,6 @@ func newTransport(config *tls.Config) *http.Transport {
 	return t
 }
 
-func encodeEndpoints(en []*Endpoint) []string {
-	var tags []string
-	for _, e := range en {
-		if b, err := json.Marshal(e); err == nil {
-			tags = append(tags, "e="+string(b))
-		}
-	}
-	return tags
-}
-
-func decodeEndpoints(tags []string) []*Endpoint {
-	var en []*Endpoint
-	for _, tag := range tags {
-		if len(tag) == 0 || tag[0] != 'e' {
-			continue
-		}
-
-		var e *Endpoint
-		if err := json.Unmarshal([]byte(tag[2:]), &e); err == nil {
-			en = append(en, e)
-		}
-	}
-	return en
-}
-
-func encodeMetadata(md map[string]string) []string {
-	var tags []string
-	for k, v := range md {
-		if b, err := json.Marshal(map[string]string{
-			k: v,
-		}); err == nil {
-			tags = append(tags, "t="+string(b))
-		}
-	}
-	return tags
-}
-
-func decodeMetadata(tags []string) map[string]string {
-	md := make(map[string]string)
-	for _, tag := range tags {
-		if len(tag) == 0 || tag[0] != 't' {
-			continue
-		}
-
-		var kv map[string]string
-		if err := json.Unmarshal([]byte(tag[2:]), &kv); err == nil {
-			for k, v := range kv {
-				md[k] = v
-			}
-		}
-	}
-	return md
-}
-
-func encodeVersion(v string) string {
-	return "v=" + v
-}
-
-func decodeVersion(tags []string) (string, bool) {
-	for _, tag := range tags {
-		if len(tag) == 0 || tag[0] != 'v' {
-			continue
-		}
-		return tag[2:], true
-	}
-	return "", false
-}
-
 func newConsulRegistry(opts ...Option) Registry {
 	var options Options
 	for _, o := range opts {
@@ -177,7 +108,7 @@ func (c *consulRegistry) Register(s *Service, opts ...RegisterOption) error {
 
 	tags := encodeMetadata(node.Metadata)
 	tags = append(tags, encodeEndpoints(s.Endpoints)...)
-	tags = append(tags, encodeVersion(s.Version))
+	tags = append(tags, encodeVersion(s.Version)...)
 
 	var check *consul.AgentServiceCheck
 

registry/encoding.go

@@ -0,0 +1,181 @@
+package registry
+
+import (
+	"bytes"
+	"compress/zlib"
+	"encoding/hex"
+	"encoding/json"
+	"io/ioutil"
+)
+
+func encode(buf []byte) string {
+	var b bytes.Buffer
+	defer b.Reset()
+
+	w := zlib.NewWriter(&b)
+	if _, err := w.Write(buf); err != nil {
+		return ""
+	}
+	w.Close()
+
+	return hex.EncodeToString(b.Bytes())
+}
+
+func decode(d string) []byte {
+	hr, err := hex.DecodeString(d)
+	if err != nil {
+		return nil
+	}
+
+	br := bytes.NewReader(hr)
+	zr, err := zlib.NewReader(br)
+	if err != nil {
+		return nil
+	}
+
+	rbuf, err := ioutil.ReadAll(zr)
+	if err != nil {
+		return nil
+	}
+
+	return rbuf
+}
+
+func encodeEndpoints(en []*Endpoint) []string {
+	var tags []string
+	for _, e := range en {
+		if b, err := json.Marshal(e); err == nil {
+			// old encoding
+			// TODO: remove in 09/2016
+			tags = append(tags, "e="+string(b))
+			// new encoding
+			tags = append(tags, "e-"+encode(b))
+		}
+	}
+	return tags
+}
+
+func decodeEndpoints(tags []string) []*Endpoint {
+	var en []*Endpoint
+
+	// use the first format you find
+	var ver byte
+
+	for _, tag := range tags {
+		if len(tag) == 0 || tag[0] != 'e' {
+			continue
+		}
+
+		// check version
+		if ver > 0 && tag[1] != ver {
+			continue
+		}
+
+		var e *Endpoint
+		var buf []byte
+
+		// Old encoding was plain
+		if tag[1] == '=' {
+			buf = []byte(tag[2:])
+		}
+
+		// New encoding is hex
+		if tag[1] == '-' {
+			buf = decode(tag[2:])
+		}
+
+		if err := json.Unmarshal(buf, &e); err == nil {
+			en = append(en, e)
+		}
+
+		// set version
+		ver = tag[1]
+	}
+	return en
+}
+
+func encodeMetadata(md map[string]string) []string {
+	var tags []string
+	for k, v := range md {
+		if b, err := json.Marshal(map[string]string{
+			k: v,
+		}); err == nil {
+			// old encoding
+			// TODO: remove in 09/2016
+			tags = append(tags, "t="+string(b))
+			// new encoding
+			tags = append(tags, "t-"+encode(b))
+		}
+	}
+	return tags
+}
+
+func decodeMetadata(tags []string) map[string]string {
+	md := make(map[string]string)
+
+	var ver byte
+
+	for _, tag := range tags {
+		if len(tag) == 0 || tag[0] != 't' {
+			continue
+		}
+
+		// check version
+		if ver > 0 && tag[1] != ver {
+			continue
+		}
+
+		var kv map[string]string
+		var buf []byte
+
+		// Old encoding was plain
+		if tag[1] == '=' {
+			buf = []byte(tag[2:])
+		}
+
+		// New encoding is hex
+		if tag[1] == '-' {
+			buf = decode(tag[2:])
+		}
+
+		// Now unmarshal
+		if err := json.Unmarshal(buf, &kv); err == nil {
+			for k, v := range kv {
+				md[k] = v
+			}
+		}
+
+		// set version
+		ver = tag[1]
+	}
+	return md
+}
+
+func encodeVersion(v string) []string {
+	return []string{
+		// old encoding,
+		// TODO: remove in 09/2016
+		"v=" + v,
+		// new encoding,
+		"v-" + encode([]byte(v)),
+	}
+}
+
+func decodeVersion(tags []string) (string, bool) {
+	for _, tag := range tags {
+		if len(tag) < 2 || tag[0] != 'v' {
+			continue
+		}
+
+		// Old encoding was plain
+		if tag[1] == '=' {
+			return tag[2:], true
+		}
+
+		// New encoding is hex
+		if tag[1] == '-' {
+			return string(decode(tag[2:])), true
+		}
+	}
+	return "", false
+}

registry/encoding_test.go

@@ -0,0 +1,161 @@
+package registry
+
+import (
+	"encoding/json"
+
+	"testing"
+)
+
+func TestEncodingEndpoints(t *testing.T) {
+	eps := []*Endpoint{
+		&Endpoint{
+			Name: "endpoint1",
+			Request: &Value{
+				Name: "request",
+				Type: "request",
+			},
+			Response: &Value{
+				Name: "response",
+				Type: "response",
+			},
+			Metadata: map[string]string{
+				"foo1": "bar1",
+			},
+		},
+		&Endpoint{
+			Name: "endpoint2",
+			Request: &Value{
+				Name: "request",
+				Type: "request",
+			},
+			Response: &Value{
+				Name: "response",
+				Type: "response",
+			},
+			Metadata: map[string]string{
+				"foo2": "bar2",
+			},
+		},
+		&Endpoint{
+			Name: "endpoint3",
+			Request: &Value{
+				Name: "request",
+				Type: "request",
+			},
+			Response: &Value{
+				Name: "response",
+				Type: "response",
+			},
+			Metadata: map[string]string{
+				"foo3": "bar3",
+			},
+		},
+	}
+
+	testEp := func(ep *Endpoint, enc string) {
+		// encode endpoint
+		e := encodeEndpoints([]*Endpoint{ep})
+
+		// check there are two tags; old and new
+		if len(e) != 2 {
+			t.Fatal("Expected 2 encoded tags, got %v", e)
+		}
+
+		// check old encoding
+		var seen bool
+
+		for _, en := range e {
+			if en == enc {
+				seen = true
+				break
+			}
+		}
+
+		if !seen {
+			t.Fatal("Expected %s but not found", enc)
+		}
+
+		// decode
+		d := decodeEndpoints([]string{enc})
+		if len(d) == 0 {
+			t.Fatalf("Expected %v got %v", ep, d)
+		}
+
+		// check name
+		if d[0].Name != ep.Name {
+			t.Fatalf("Expected ep %s got %s", ep.Name, d[0].Name)
+		}
+
+		// check all the metadata exists
+		for k, v := range ep.Metadata {
+			if gv := d[0].Metadata[k]; gv != v {
+				t.Fatalf("Expected key %s val %s got val %s", k, v, gv)
+			}
+		}
+	}
+
+	for _, ep := range eps {
+		// JSON encoded
+		jencoded, err := json.Marshal(ep)
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		// HEX encoded
+		hencoded := encode(jencoded)
+		// endpoint tag
+		jepTag := "e=" + string(jencoded)
+		hepTag := "e-" + hencoded
+
+		// test old
+		testEp(ep, jepTag)
+		// test new
+		testEp(ep, hepTag)
+	}
+}
+
+func TestEncodingVersion(t *testing.T) {
+	testData := []struct {
+		decoded    string
+		encoded    string
+		oldEncoded string
+	}{
+		{"1.0.0", "v-789c32d433d03300040000ffff02ce00ee", "v=1.0.0"},
+		{"latest", "v-789cca492c492d2e01040000ffff08cc028e", "v=latest"},
+	}
+
+	for _, data := range testData {
+		e := encodeVersion(data.decoded)
+
+		if e[1] != data.encoded {
+			t.Fatalf("Expected %s got %s", data.encoded, e)
+		}
+
+		d, ok := decodeVersion(e)
+		if !ok {
+			t.Fatal("Unexpected %t for %s", ok, data.encoded)
+		}
+
+		if d != data.decoded {
+			t.Fatal("Expected %s got %s", data.decoded, d)
+		}
+
+		d, ok = decodeVersion([]string{data.encoded})
+		if !ok {
+			t.Fatal("Unexpected %t for %s", ok, data.encoded)
+		}
+
+		if d != data.decoded {
+			t.Fatal("Expected %s got %s", data.decoded, d)
+		}
+
+		d, ok = decodeVersion([]string{data.oldEncoded})
+		if !ok {
+			t.Fatal("Unexpected %t for %s", ok, data.oldEncoded)
+		}
+
+		if d != data.decoded {
+			t.Fatal("Expected %s got %s", data.decoded, d)
+		}
+	}
+}