bundle qson lib in util (#1561)

* copy qson from https://github.com/joncalhoun/qson
  as author not want to maintain repo
* latest code contains our fix to proper decode strings
  with escaped & symbol
* replace package in api/handler/rpc

Signed-off-by: Vasiliy Tolstov <v.tolstov@unistack.org>

api/handler/rpc/rpc.go

@@ -10,7 +10,6 @@ import (
 	"strings"
 
 	jsonpatch "github.com/evanphx/json-patch/v5"
-	"github.com/joncalhoun/qson"
 	"github.com/micro/go-micro/v2/api"
 	"github.com/micro/go-micro/v2/api/handler"
 	"github.com/micro/go-micro/v2/api/internal/proto"
@@ -24,6 +23,7 @@ import (
 	"github.com/micro/go-micro/v2/metadata"
 	"github.com/micro/go-micro/v2/registry"
 	"github.com/micro/go-micro/v2/util/ctx"
+	"github.com/micro/go-micro/v2/util/qson"
 	"github.com/oxtoacart/bpool"
 )
 

go.mod

@@ -39,7 +39,6 @@ require (
 	github.com/hpcloud/tail v1.0.0
 	github.com/imdario/mergo v0.3.8
 	github.com/jonboulle/clockwork v0.1.0 // indirect
-	github.com/joncalhoun/qson v0.0.0-20170526102502-8a9cab3a62b1
 	github.com/json-iterator/go v1.1.9 // indirect
 	github.com/kr/pretty v0.1.0
 	github.com/lib/pq v1.3.0

go.sum

@@ -238,8 +238,6 @@ github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJS
 github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k=
 github.com/jonboulle/clockwork v0.1.0 h1:VKV+ZcuP6l3yW9doeqz6ziZGgcynBVQO+obU0+0hcPo=
 github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo=
-github.com/joncalhoun/qson v0.0.0-20170526102502-8a9cab3a62b1 h1:lnrOS18wZBYrzdDmnUeg1OVk+kQ3rxG8mZWU89DpMIA=
-github.com/joncalhoun/qson v0.0.0-20170526102502-8a9cab3a62b1/go.mod h1:DFXrEwSRX0p/aSvxE21319menCBFeQO0jXpRj7LEZUA=
 github.com/json-iterator/go v1.1.5/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
 github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
 github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=

util/qson/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2020 Jon Calhoun
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

util/qson/README.md

@@ -0,0 +1,55 @@
+# qson
+
+This is copy from https://github.com/joncalhoun/qson
+As author says he is not acrivelly maintains the repo and not plan to do that.
+
+## Usage
+
+You can either turn a URL query param into a JSON byte array, or unmarshal that directly into a Go object.
+
+Transforming the URL query param into a JSON byte array:
+
+```go
+import "github.com/joncalhoun/qson"
+
+func main() {
+  b, err := qson.ToJSON("bar%5Bone%5D%5Btwo%5D=2&bar[one][red]=112")
+  if err != nil {
+    panic(err)
+  }
+  fmt.Println(string(b))
+  // Should output: {"bar":{"one":{"red":112,"two":2}}}
+}
+```
+
+Or unmarshalling directly into a Go object using JSON struct tags:
+
+```go
+import "github.com/joncalhoun/qson"
+
+type unmarshalT struct {
+	A string     `json:"a"`
+	B unmarshalB `json:"b"`
+}
+type unmarshalB struct {
+	C int `json:"c"`
+}
+
+func main() {
+  var out unmarshalT
+  query := "a=xyz&b[c]=456"
+  err := Unmarshal(&out, query)
+  if err != nil {
+  	t.Error(err)
+  }
+  // out should equal
+  //   unmarshalT{
+	// 	  A: "xyz",
+	// 	  B: unmarshalB{
+	// 	  	C: 456,
+	// 	  },
+	//   }
+}
+```
+
+To get a query string like in the two previous examples you can use the `RawQuery` field on the [net/url.URL](https://golang.org/pkg/net/url/#URL) type.

util/qson/merge.go

@@ -0,0 +1,34 @@
+package qson
+
+// merge merges a with b if they are either both slices
+// or map[string]interface{} types. Otherwise it returns b.
+func merge(a interface{}, b interface{}) interface{} {
+	switch aT := a.(type) {
+	case map[string]interface{}:
+		return mergeMap(aT, b.(map[string]interface{}))
+	case []interface{}:
+		return mergeSlice(aT, b.([]interface{}))
+	default:
+		return b
+	}
+}
+
+// mergeMap merges a with b, attempting to merge any nested
+// values in nested maps but eventually overwriting anything
+// in a that can't be merged with whatever is in b.
+func mergeMap(a map[string]interface{}, b map[string]interface{}) map[string]interface{} {
+	for bK, bV := range b {
+		if _, ok := a[bK]; ok {
+			a[bK] = merge(a[bK], bV)
+		} else {
+			a[bK] = bV
+		}
+	}
+	return a
+}
+
+// mergeSlice merges a with b and returns the result.
+func mergeSlice(a []interface{}, b []interface{}) []interface{} {
+	a = append(a, b...)
+	return a
+}

util/qson/merge_test.go

@@ -0,0 +1,37 @@
+package qson
+
+import "testing"
+
+func TestMergeSlice(t *testing.T) {
+	a := []interface{}{"a"}
+	b := []interface{}{"b"}
+	actual := mergeSlice(a, b)
+	if len(actual) != 2 {
+		t.Errorf("Expected size to be 2.")
+	}
+	if actual[0] != "a" {
+		t.Errorf("Expected index 0 to have value a. Actual: %s", actual[0])
+	}
+	if actual[1] != "b" {
+		t.Errorf("Expected index 1 to have value b. Actual: %s", actual[1])
+	}
+}
+
+func TestMergeMap(t *testing.T) {
+	a := map[string]interface{}{
+		"a": "b",
+	}
+	b := map[string]interface{}{
+		"b": "c",
+	}
+	actual := mergeMap(a, b)
+	if len(actual) != 2 {
+		t.Errorf("Expected size to be 2.")
+	}
+	if actual["a"] != "b" {
+		t.Errorf("Expected key \"a\" to have value b. Actual: %s", actual["a"])
+	}
+	if actual["b"] != "c" {
+		t.Errorf("Expected key \"b\" to have value c. Actual: %s", actual["b"])
+	}
+}

util/qson/qson.go

@@ -0,0 +1,154 @@
+// Package qson implmenets decoding of URL query params
+// into JSON and Go values (using JSON struct tags).
+//
+// See https://golang.org/pkg/encoding/json/ for more
+// details on JSON struct tags.
+package qson
+
+import (
+	"encoding/json"
+	"errors"
+	"net/url"
+	"regexp"
+	"strings"
+)
+
+var (
+	// ErrInvalidParam is returned when invalid data is provided to the ToJSON or Unmarshal function.
+	// Specifically, this will be returned when there is no equals sign present in the URL query parameter.
+	ErrInvalidParam error = errors.New("qson: invalid url query param provided")
+
+	bracketSplitter *regexp.Regexp
+)
+
+func init() {
+	bracketSplitter = regexp.MustCompile("\\[|\\]")
+}
+
+// Unmarshal will take a dest along with URL
+// query params and attempt to first turn the query params
+// into JSON and then unmarshal those into the dest variable
+//
+// BUG(joncalhoun): If a URL query param value is something
+// like 123 but is expected to be parsed into a string this
+// will currently result in an error because the JSON
+// transformation will assume this is intended to be an int.
+// This should only affect the Unmarshal function and
+// could likely be fixed, but someone will need to submit a
+// PR if they want that fixed.
+func Unmarshal(dst interface{}, query string) error {
+	b, err := ToJSON(query)
+	if err != nil {
+		return err
+	}
+	return json.Unmarshal(b, dst)
+}
+
+// ToJSON will turn a query string like:
+//   cat=1&bar%5Bone%5D%5Btwo%5D=2&bar[one][red]=112
+// Into a JSON object with all the data merged as nicely as
+// possible. Eg the example above would output:
+//   {"bar":{"one":{"two":2,"red":112}}}
+func ToJSON(query string) ([]byte, error) {
+	var (
+		builder interface{} = make(map[string]interface{})
+	)
+	params := strings.Split(query, "&")
+	for _, part := range params {
+		tempMap, err := queryToMap(part)
+		if err != nil {
+			return nil, err
+		}
+		builder = merge(builder, tempMap)
+	}
+	return json.Marshal(builder)
+}
+
+// queryToMap turns something like a[b][c]=4 into
+//   map[string]interface{}{
+//     "a": map[string]interface{}{
+// 		  "b": map[string]interface{}{
+// 			  "c": 4,
+// 		  },
+// 	  },
+//   }
+func queryToMap(param string) (map[string]interface{}, error) {
+	rawKey, rawValue, err := splitKeyAndValue(param)
+	if err != nil {
+		return nil, err
+	}
+	rawValue, err = url.QueryUnescape(rawValue)
+	if err != nil {
+		return nil, err
+	}
+	rawKey, err = url.QueryUnescape(rawKey)
+	if err != nil {
+		return nil, err
+	}
+
+	pieces := bracketSplitter.Split(rawKey, -1)
+	key := pieces[0]
+
+	// If len==1 then rawKey has no [] chars and we can just
+	// decode this as key=value into {key: value}
+	if len(pieces) == 1 {
+		var value interface{}
+		// First we try parsing it as an int, bool, null, etc
+		err = json.Unmarshal([]byte(rawValue), &value)
+		if err != nil {
+			// If we got an error we try wrapping the value in
+			// quotes and processing it as a string
+			err = json.Unmarshal([]byte("\""+rawValue+"\""), &value)
+			if err != nil {
+				// If we can't decode as a string we return the err
+				return nil, err
+			}
+		}
+		return map[string]interface{}{
+			key: value,
+		}, nil
+	}
+
+	// If len > 1 then we have something like a[b][c]=2
+	// so we need to turn this into {"a": {"b": {"c": 2}}}
+	// To do this we break our key into two pieces:
+	//   a and b[c]
+	// and then we set {"a": queryToMap("b[c]", value)}
+	ret := make(map[string]interface{}, 0)
+	ret[key], err = queryToMap(buildNewKey(rawKey) + "=" + rawValue)
+	if err != nil {
+		return nil, err
+	}
+
+	// When URL params have a set of empty brackets (eg a[]=1)
+	// it is assumed to be an array. This will get us the
+	// correct value for the array item and return it as an
+	// []interface{} so that it can be merged properly.
+	if pieces[1] == "" {
+		temp := ret[key].(map[string]interface{})
+		ret[key] = []interface{}{temp[""]}
+	}
+	return ret, nil
+}
+
+// buildNewKey will take something like:
+// origKey = "bar[one][two]"
+// pieces = [bar one two ]
+// and return "one[two]"
+func buildNewKey(origKey string) string {
+	pieces := bracketSplitter.Split(origKey, -1)
+	ret := origKey[len(pieces[0])+1:]
+	ret = ret[:len(pieces[1])] + ret[len(pieces[1])+1:]
+	return ret
+}
+
+// splitKeyAndValue splits a URL param at the last equal
+// sign and returns the two strings. If no equal sign is
+// found, the ErrInvalidParam error is returned.
+func splitKeyAndValue(param string) (string, string, error) {
+	li := strings.LastIndex(param, "=")
+	if li == -1 {
+		return "", "", ErrInvalidParam
+	}
+	return param[:li], param[li+1:], nil
+}

util/qson/qson_test.go

@@ -0,0 +1,170 @@
+package qson
+
+import (
+	"fmt"
+	"testing"
+)
+
+func ExampleUnmarshal() {
+	type Ex struct {
+		A string `json:"a"`
+		B struct {
+			C int `json:"c"`
+		} `json:"b"`
+	}
+	var ex Ex
+	if err := Unmarshal(&ex, "a=xyz&b[c]=456"); err != nil {
+		panic(err)
+	}
+	fmt.Printf("%+v\n", ex)
+	// Output: {A:xyz B:{C:456}}
+}
+
+type unmarshalT struct {
+	A string     `json:"a"`
+	B unmarshalB `json:"b"`
+}
+type unmarshalB struct {
+	C int    `json:"c"`
+	D string `json:"D"`
+}
+
+func TestUnmarshal(t *testing.T) {
+	query := "a=xyz&b[c]=456"
+	expected := unmarshalT{
+		A: "xyz",
+		B: unmarshalB{
+			C: 456,
+		},
+	}
+	var actual unmarshalT
+	err := Unmarshal(&actual, query)
+	if err != nil {
+		t.Error(err)
+	}
+	if expected != actual {
+		t.Errorf("Expected: %+v Actual: %+v", expected, actual)
+	}
+}
+
+func ExampleToJSON() {
+	b, err := ToJSON("a=xyz&b[c]=456")
+	if err != nil {
+		panic(err)
+	}
+	fmt.Printf(string(b))
+	// Output: {"a":"xyz","b":{"c":456}}
+}
+
+func TestToJSONNested(t *testing.T) {
+	query := "bar%5Bone%5D%5Btwo%5D=2&bar[one][red]=112"
+	expected := `{"bar":{"one":{"red":112,"two":2}}}`
+	actual, err := ToJSON(query)
+	if err != nil {
+		t.Error(err)
+	}
+	actualStr := string(actual)
+	if actualStr != expected {
+		t.Errorf("Expected: %s Actual: %s", expected, actualStr)
+	}
+}
+
+func TestToJSONPlain(t *testing.T) {
+	query := "cat=1&dog=2"
+	expected := `{"cat":1,"dog":2}`
+	actual, err := ToJSON(query)
+	if err != nil {
+		t.Error(err)
+	}
+	actualStr := string(actual)
+	if actualStr != expected {
+		t.Errorf("Expected: %s Actual: %s", expected, actualStr)
+	}
+}
+
+func TestToJSONSlice(t *testing.T) {
+	query := "cat[]=1&cat[]=34"
+	expected := `{"cat":[1,34]}`
+	actual, err := ToJSON(query)
+	if err != nil {
+		t.Error(err)
+	}
+	actualStr := string(actual)
+	if actualStr != expected {
+		t.Errorf("Expected: %s Actual: %s", expected, actualStr)
+	}
+}
+
+func TestToJSONBig(t *testing.T) {
+	query := "distinct_id=763_1495187301909_3495&timestamp=1495187523&event=product_add_cart&params%5BproductRefId%5D=8284563078&params%5Bapps%5D%5B%5D=precommend&params%5Bapps%5D%5B%5D=bsales&params%5Bsource%5D=item&params%5Boptions%5D%5Bsegment%5D=cart_recommendation&params%5Boptions%5D%5Btype%5D=up_sell&params%5BtimeExpire%5D=1495187599642&params%5Brecommend_system_product_source%5D=item&params%5Bproduct_id%5D=8284563078&params%5Bvariant_id%5D=27661944134&params%5Bsku%5D=00483332%20(black)&params%5Bsources%5D%5B%5D=product_recommendation&params%5Bcart_token%5D=dc2c336a009edf2762128e65806dfb1d&params%5Bquantity%5D=1&params%5Bnew_popup_upsell_mobile%5D=false&params%5BclientDevice%5D=desktop&params%5BclientIsMobile%5D=false&params%5BclientIsSmallScreen%5D=false&params%5Bnew_popup_crossell_mobile%5D=false&api_key=14c5b7dacea9157029265b174491d340"
+	expected := `{"api_key":"14c5b7dacea9157029265b174491d340","distinct_id":"763_1495187301909_3495","event":"product_add_cart","params":{"apps":["precommend","bsales"],"cart_token":"dc2c336a009edf2762128e65806dfb1d","clientDevice":"desktop","clientIsMobile":false,"clientIsSmallScreen":false,"new_popup_crossell_mobile":false,"new_popup_upsell_mobile":false,"options":{"segment":"cart_recommendation","type":"up_sell"},"productRefId":8284563078,"product_id":8284563078,"quantity":1,"recommend_system_product_source":"item","sku":"00483332 (black)","source":"item","sources":["product_recommendation"],"timeExpire":1495187599642,"variant_id":27661944134},"timestamp":1495187523}`
+	actual, err := ToJSON(query)
+	if err != nil {
+		t.Error(err)
+	}
+	actualStr := string(actual)
+	if actualStr != expected {
+		t.Errorf("Expected: %s Actual: %s", expected, actualStr)
+	}
+}
+
+func TestToJSONDuplicateKey(t *testing.T) {
+	query := "cat=1&cat=2"
+	expected := `{"cat":2}`
+	actual, err := ToJSON(query)
+	if err != nil {
+		t.Error(err)
+	}
+	actualStr := string(actual)
+	if actualStr != expected {
+		t.Errorf("Expected: %s Actual: %s", expected, actualStr)
+	}
+}
+
+func TestSplitKeyAndValue(t *testing.T) {
+	param := "a[dog][=cat]=123"
+	eKey, eValue := "a[dog][=cat]", "123"
+	aKey, aValue, err := splitKeyAndValue(param)
+	if err != nil {
+		t.Error(err)
+	}
+	if eKey != aKey {
+		t.Errorf("Keys do not match. Expected: %s Actual: %s", eKey, aKey)
+	}
+	if eValue != aValue {
+		t.Errorf("Values do not match. Expected: %s Actual: %s", eValue, aValue)
+	}
+}
+
+func TestEncodedAmpersand(t *testing.T) {
+	query := "a=xyz&b[d]=ben%26jerry"
+	expected := unmarshalT{
+		A: "xyz",
+		B: unmarshalB{
+			D: "ben&jerry",
+		},
+	}
+	var actual unmarshalT
+	err := Unmarshal(&actual, query)
+	if err != nil {
+		t.Error(err)
+	}
+	if expected != actual {
+		t.Errorf("Expected: %+v Actual: %+v", expected, actual)
+	}
+}
+
+func TestEncodedAmpersand2(t *testing.T) {
+	query := "filter=parent%3Dflow12345%26request%3Dreq12345&meta.limit=20&meta.offset=0"
+	expected := map[string]interface{}{"filter": "parent=flow12345&request=req12345", "meta.limit": float64(20), "meta.offset": float64(0)}
+	actual := make(map[string]interface{})
+	err := Unmarshal(&actual, query)
+	if err != nil {
+		t.Error(err)
+	}
+	for k, v := range actual {
+		if nv, ok := expected[k]; !ok || nv != v {
+			t.Errorf("Expected: %+v Actual: %+v", expected, actual)
+		}
+	}
+}