From 6fa27373edc2175556ebb2010d2f9374873ded92 Mon Sep 17 00:00:00 2001 From: Vasiliy Tolstov Date: Thu, 23 Apr 2020 11:08:09 +0300 Subject: [PATCH] bundle qson lib in util (#1561) * copy qson from https://github.com/joncalhoun/qson as author not want to maintain repo * latest code contains our fix to proper decode strings with escaped & symbol * replace package in api/handler/rpc Signed-off-by: Vasiliy Tolstov --- api/handler/rpc/rpc.go | 2 +- go.mod | 1 - go.sum | 2 - util/qson/LICENSE | 21 +++++ util/qson/README.md | 55 +++++++++++++ util/qson/merge.go | 34 ++++++++ util/qson/merge_test.go | 37 +++++++++ util/qson/qson.go | 154 ++++++++++++++++++++++++++++++++++++ util/qson/qson_test.go | 170 ++++++++++++++++++++++++++++++++++++++++ 9 files changed, 472 insertions(+), 4 deletions(-) create mode 100644 util/qson/LICENSE create mode 100644 util/qson/README.md create mode 100644 util/qson/merge.go create mode 100644 util/qson/merge_test.go create mode 100644 util/qson/qson.go create mode 100644 util/qson/qson_test.go diff --git a/api/handler/rpc/rpc.go b/api/handler/rpc/rpc.go index 64f0cf48..542a9dd4 100644 --- a/api/handler/rpc/rpc.go +++ b/api/handler/rpc/rpc.go @@ -10,7 +10,6 @@ import ( "strings" jsonpatch "github.com/evanphx/json-patch/v5" - "github.com/joncalhoun/qson" "github.com/micro/go-micro/v2/api" "github.com/micro/go-micro/v2/api/handler" "github.com/micro/go-micro/v2/api/internal/proto" @@ -24,6 +23,7 @@ import ( "github.com/micro/go-micro/v2/metadata" "github.com/micro/go-micro/v2/registry" "github.com/micro/go-micro/v2/util/ctx" + "github.com/micro/go-micro/v2/util/qson" "github.com/oxtoacart/bpool" ) diff --git a/go.mod b/go.mod index f3ba49cd..05e82553 100644 --- a/go.mod +++ b/go.mod @@ -39,7 +39,6 @@ require ( github.com/hpcloud/tail v1.0.0 github.com/imdario/mergo v0.3.8 github.com/jonboulle/clockwork v0.1.0 // indirect - github.com/joncalhoun/qson v0.0.0-20170526102502-8a9cab3a62b1 github.com/json-iterator/go v1.1.9 // indirect github.com/kr/pretty v0.1.0 github.com/lib/pq v1.3.0 diff --git a/go.sum b/go.sum index db9453b6..a204a929 100644 --- a/go.sum +++ b/go.sum @@ -238,8 +238,6 @@ github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJS github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jonboulle/clockwork v0.1.0 h1:VKV+ZcuP6l3yW9doeqz6ziZGgcynBVQO+obU0+0hcPo= github.com/jonboulle/clockwork v0.1.0/go.mod h1:Ii8DK3G1RaLaWxj9trq07+26W01tbo22gdxWY5EU2bo= -github.com/joncalhoun/qson v0.0.0-20170526102502-8a9cab3a62b1 h1:lnrOS18wZBYrzdDmnUeg1OVk+kQ3rxG8mZWU89DpMIA= -github.com/joncalhoun/qson v0.0.0-20170526102502-8a9cab3a62b1/go.mod h1:DFXrEwSRX0p/aSvxE21319menCBFeQO0jXpRj7LEZUA= github.com/json-iterator/go v1.1.5/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= diff --git a/util/qson/LICENSE b/util/qson/LICENSE new file mode 100644 index 00000000..3e4ba4f7 --- /dev/null +++ b/util/qson/LICENSE @@ -0,0 +1,21 @@ +MIT License + +Copyright (c) 2020 Jon Calhoun + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/util/qson/README.md b/util/qson/README.md new file mode 100644 index 00000000..ad76ced5 --- /dev/null +++ b/util/qson/README.md @@ -0,0 +1,55 @@ +# qson + +This is copy from https://github.com/joncalhoun/qson +As author says he is not acrivelly maintains the repo and not plan to do that. + +## Usage + +You can either turn a URL query param into a JSON byte array, or unmarshal that directly into a Go object. + +Transforming the URL query param into a JSON byte array: + +```go +import "github.com/joncalhoun/qson" + +func main() { + b, err := qson.ToJSON("bar%5Bone%5D%5Btwo%5D=2&bar[one][red]=112") + if err != nil { + panic(err) + } + fmt.Println(string(b)) + // Should output: {"bar":{"one":{"red":112,"two":2}}} +} +``` + +Or unmarshalling directly into a Go object using JSON struct tags: + +```go +import "github.com/joncalhoun/qson" + +type unmarshalT struct { + A string `json:"a"` + B unmarshalB `json:"b"` +} +type unmarshalB struct { + C int `json:"c"` +} + +func main() { + var out unmarshalT + query := "a=xyz&b[c]=456" + err := Unmarshal(&out, query) + if err != nil { + t.Error(err) + } + // out should equal + // unmarshalT{ + // A: "xyz", + // B: unmarshalB{ + // C: 456, + // }, + // } +} +``` + +To get a query string like in the two previous examples you can use the `RawQuery` field on the [net/url.URL](https://golang.org/pkg/net/url/#URL) type. diff --git a/util/qson/merge.go b/util/qson/merge.go new file mode 100644 index 00000000..64078078 --- /dev/null +++ b/util/qson/merge.go @@ -0,0 +1,34 @@ +package qson + +// merge merges a with b if they are either both slices +// or map[string]interface{} types. Otherwise it returns b. +func merge(a interface{}, b interface{}) interface{} { + switch aT := a.(type) { + case map[string]interface{}: + return mergeMap(aT, b.(map[string]interface{})) + case []interface{}: + return mergeSlice(aT, b.([]interface{})) + default: + return b + } +} + +// mergeMap merges a with b, attempting to merge any nested +// values in nested maps but eventually overwriting anything +// in a that can't be merged with whatever is in b. +func mergeMap(a map[string]interface{}, b map[string]interface{}) map[string]interface{} { + for bK, bV := range b { + if _, ok := a[bK]; ok { + a[bK] = merge(a[bK], bV) + } else { + a[bK] = bV + } + } + return a +} + +// mergeSlice merges a with b and returns the result. +func mergeSlice(a []interface{}, b []interface{}) []interface{} { + a = append(a, b...) + return a +} diff --git a/util/qson/merge_test.go b/util/qson/merge_test.go new file mode 100644 index 00000000..9a144db8 --- /dev/null +++ b/util/qson/merge_test.go @@ -0,0 +1,37 @@ +package qson + +import "testing" + +func TestMergeSlice(t *testing.T) { + a := []interface{}{"a"} + b := []interface{}{"b"} + actual := mergeSlice(a, b) + if len(actual) != 2 { + t.Errorf("Expected size to be 2.") + } + if actual[0] != "a" { + t.Errorf("Expected index 0 to have value a. Actual: %s", actual[0]) + } + if actual[1] != "b" { + t.Errorf("Expected index 1 to have value b. Actual: %s", actual[1]) + } +} + +func TestMergeMap(t *testing.T) { + a := map[string]interface{}{ + "a": "b", + } + b := map[string]interface{}{ + "b": "c", + } + actual := mergeMap(a, b) + if len(actual) != 2 { + t.Errorf("Expected size to be 2.") + } + if actual["a"] != "b" { + t.Errorf("Expected key \"a\" to have value b. Actual: %s", actual["a"]) + } + if actual["b"] != "c" { + t.Errorf("Expected key \"b\" to have value c. Actual: %s", actual["b"]) + } +} diff --git a/util/qson/qson.go b/util/qson/qson.go new file mode 100644 index 00000000..b3926167 --- /dev/null +++ b/util/qson/qson.go @@ -0,0 +1,154 @@ +// Package qson implmenets decoding of URL query params +// into JSON and Go values (using JSON struct tags). +// +// See https://golang.org/pkg/encoding/json/ for more +// details on JSON struct tags. +package qson + +import ( + "encoding/json" + "errors" + "net/url" + "regexp" + "strings" +) + +var ( + // ErrInvalidParam is returned when invalid data is provided to the ToJSON or Unmarshal function. + // Specifically, this will be returned when there is no equals sign present in the URL query parameter. + ErrInvalidParam error = errors.New("qson: invalid url query param provided") + + bracketSplitter *regexp.Regexp +) + +func init() { + bracketSplitter = regexp.MustCompile("\\[|\\]") +} + +// Unmarshal will take a dest along with URL +// query params and attempt to first turn the query params +// into JSON and then unmarshal those into the dest variable +// +// BUG(joncalhoun): If a URL query param value is something +// like 123 but is expected to be parsed into a string this +// will currently result in an error because the JSON +// transformation will assume this is intended to be an int. +// This should only affect the Unmarshal function and +// could likely be fixed, but someone will need to submit a +// PR if they want that fixed. +func Unmarshal(dst interface{}, query string) error { + b, err := ToJSON(query) + if err != nil { + return err + } + return json.Unmarshal(b, dst) +} + +// ToJSON will turn a query string like: +// cat=1&bar%5Bone%5D%5Btwo%5D=2&bar[one][red]=112 +// Into a JSON object with all the data merged as nicely as +// possible. Eg the example above would output: +// {"bar":{"one":{"two":2,"red":112}}} +func ToJSON(query string) ([]byte, error) { + var ( + builder interface{} = make(map[string]interface{}) + ) + params := strings.Split(query, "&") + for _, part := range params { + tempMap, err := queryToMap(part) + if err != nil { + return nil, err + } + builder = merge(builder, tempMap) + } + return json.Marshal(builder) +} + +// queryToMap turns something like a[b][c]=4 into +// map[string]interface{}{ +// "a": map[string]interface{}{ +// "b": map[string]interface{}{ +// "c": 4, +// }, +// }, +// } +func queryToMap(param string) (map[string]interface{}, error) { + rawKey, rawValue, err := splitKeyAndValue(param) + if err != nil { + return nil, err + } + rawValue, err = url.QueryUnescape(rawValue) + if err != nil { + return nil, err + } + rawKey, err = url.QueryUnescape(rawKey) + if err != nil { + return nil, err + } + + pieces := bracketSplitter.Split(rawKey, -1) + key := pieces[0] + + // If len==1 then rawKey has no [] chars and we can just + // decode this as key=value into {key: value} + if len(pieces) == 1 { + var value interface{} + // First we try parsing it as an int, bool, null, etc + err = json.Unmarshal([]byte(rawValue), &value) + if err != nil { + // If we got an error we try wrapping the value in + // quotes and processing it as a string + err = json.Unmarshal([]byte("\""+rawValue+"\""), &value) + if err != nil { + // If we can't decode as a string we return the err + return nil, err + } + } + return map[string]interface{}{ + key: value, + }, nil + } + + // If len > 1 then we have something like a[b][c]=2 + // so we need to turn this into {"a": {"b": {"c": 2}}} + // To do this we break our key into two pieces: + // a and b[c] + // and then we set {"a": queryToMap("b[c]", value)} + ret := make(map[string]interface{}, 0) + ret[key], err = queryToMap(buildNewKey(rawKey) + "=" + rawValue) + if err != nil { + return nil, err + } + + // When URL params have a set of empty brackets (eg a[]=1) + // it is assumed to be an array. This will get us the + // correct value for the array item and return it as an + // []interface{} so that it can be merged properly. + if pieces[1] == "" { + temp := ret[key].(map[string]interface{}) + ret[key] = []interface{}{temp[""]} + } + return ret, nil +} + +// buildNewKey will take something like: +// origKey = "bar[one][two]" +// pieces = [bar one two ] +// and return "one[two]" +func buildNewKey(origKey string) string { + pieces := bracketSplitter.Split(origKey, -1) + ret := origKey[len(pieces[0])+1:] + ret = ret[:len(pieces[1])] + ret[len(pieces[1])+1:] + return ret +} + +// splitKeyAndValue splits a URL param at the last equal +// sign and returns the two strings. If no equal sign is +// found, the ErrInvalidParam error is returned. +func splitKeyAndValue(param string) (string, string, error) { + li := strings.LastIndex(param, "=") + if li == -1 { + return "", "", ErrInvalidParam + } + return param[:li], param[li+1:], nil +} diff --git a/util/qson/qson_test.go b/util/qson/qson_test.go new file mode 100644 index 00000000..491fe0e7 --- /dev/null +++ b/util/qson/qson_test.go @@ -0,0 +1,170 @@ +package qson + +import ( + "fmt" + "testing" +) + +func ExampleUnmarshal() { + type Ex struct { + A string `json:"a"` + B struct { + C int `json:"c"` + } `json:"b"` + } + var ex Ex + if err := Unmarshal(&ex, "a=xyz&b[c]=456"); err != nil { + panic(err) + } + fmt.Printf("%+v\n", ex) + // Output: {A:xyz B:{C:456}} +} + +type unmarshalT struct { + A string `json:"a"` + B unmarshalB `json:"b"` +} +type unmarshalB struct { + C int `json:"c"` + D string `json:"D"` +} + +func TestUnmarshal(t *testing.T) { + query := "a=xyz&b[c]=456" + expected := unmarshalT{ + A: "xyz", + B: unmarshalB{ + C: 456, + }, + } + var actual unmarshalT + err := Unmarshal(&actual, query) + if err != nil { + t.Error(err) + } + if expected != actual { + t.Errorf("Expected: %+v Actual: %+v", expected, actual) + } +} + +func ExampleToJSON() { + b, err := ToJSON("a=xyz&b[c]=456") + if err != nil { + panic(err) + } + fmt.Printf(string(b)) + // Output: {"a":"xyz","b":{"c":456}} +} + +func TestToJSONNested(t *testing.T) { + query := "bar%5Bone%5D%5Btwo%5D=2&bar[one][red]=112" + expected := `{"bar":{"one":{"red":112,"two":2}}}` + actual, err := ToJSON(query) + if err != nil { + t.Error(err) + } + actualStr := string(actual) + if actualStr != expected { + t.Errorf("Expected: %s Actual: %s", expected, actualStr) + } +} + +func TestToJSONPlain(t *testing.T) { + query := "cat=1&dog=2" + expected := `{"cat":1,"dog":2}` + actual, err := ToJSON(query) + if err != nil { + t.Error(err) + } + actualStr := string(actual) + if actualStr != expected { + t.Errorf("Expected: %s Actual: %s", expected, actualStr) + } +} + +func TestToJSONSlice(t *testing.T) { + query := "cat[]=1&cat[]=34" + expected := `{"cat":[1,34]}` + actual, err := ToJSON(query) + if err != nil { + t.Error(err) + } + actualStr := string(actual) + if actualStr != expected { + t.Errorf("Expected: %s Actual: %s", expected, actualStr) + } +} + +func TestToJSONBig(t *testing.T) { + query := "distinct_id=763_1495187301909_3495×tamp=1495187523&event=product_add_cart¶ms%5BproductRefId%5D=8284563078¶ms%5Bapps%5D%5B%5D=precommend¶ms%5Bapps%5D%5B%5D=bsales¶ms%5Bsource%5D=item¶ms%5Boptions%5D%5Bsegment%5D=cart_recommendation¶ms%5Boptions%5D%5Btype%5D=up_sell¶ms%5BtimeExpire%5D=1495187599642¶ms%5Brecommend_system_product_source%5D=item¶ms%5Bproduct_id%5D=8284563078¶ms%5Bvariant_id%5D=27661944134¶ms%5Bsku%5D=00483332%20(black)¶ms%5Bsources%5D%5B%5D=product_recommendation¶ms%5Bcart_token%5D=dc2c336a009edf2762128e65806dfb1d¶ms%5Bquantity%5D=1¶ms%5Bnew_popup_upsell_mobile%5D=false¶ms%5BclientDevice%5D=desktop¶ms%5BclientIsMobile%5D=false¶ms%5BclientIsSmallScreen%5D=false¶ms%5Bnew_popup_crossell_mobile%5D=false&api_key=14c5b7dacea9157029265b174491d340" + expected := `{"api_key":"14c5b7dacea9157029265b174491d340","distinct_id":"763_1495187301909_3495","event":"product_add_cart","params":{"apps":["precommend","bsales"],"cart_token":"dc2c336a009edf2762128e65806dfb1d","clientDevice":"desktop","clientIsMobile":false,"clientIsSmallScreen":false,"new_popup_crossell_mobile":false,"new_popup_upsell_mobile":false,"options":{"segment":"cart_recommendation","type":"up_sell"},"productRefId":8284563078,"product_id":8284563078,"quantity":1,"recommend_system_product_source":"item","sku":"00483332 (black)","source":"item","sources":["product_recommendation"],"timeExpire":1495187599642,"variant_id":27661944134},"timestamp":1495187523}` + actual, err := ToJSON(query) + if err != nil { + t.Error(err) + } + actualStr := string(actual) + if actualStr != expected { + t.Errorf("Expected: %s Actual: %s", expected, actualStr) + } +} + +func TestToJSONDuplicateKey(t *testing.T) { + query := "cat=1&cat=2" + expected := `{"cat":2}` + actual, err := ToJSON(query) + if err != nil { + t.Error(err) + } + actualStr := string(actual) + if actualStr != expected { + t.Errorf("Expected: %s Actual: %s", expected, actualStr) + } +} + +func TestSplitKeyAndValue(t *testing.T) { + param := "a[dog][=cat]=123" + eKey, eValue := "a[dog][=cat]", "123" + aKey, aValue, err := splitKeyAndValue(param) + if err != nil { + t.Error(err) + } + if eKey != aKey { + t.Errorf("Keys do not match. Expected: %s Actual: %s", eKey, aKey) + } + if eValue != aValue { + t.Errorf("Values do not match. Expected: %s Actual: %s", eValue, aValue) + } +} + +func TestEncodedAmpersand(t *testing.T) { + query := "a=xyz&b[d]=ben%26jerry" + expected := unmarshalT{ + A: "xyz", + B: unmarshalB{ + D: "ben&jerry", + }, + } + var actual unmarshalT + err := Unmarshal(&actual, query) + if err != nil { + t.Error(err) + } + if expected != actual { + t.Errorf("Expected: %+v Actual: %+v", expected, actual) + } +} + +func TestEncodedAmpersand2(t *testing.T) { + query := "filter=parent%3Dflow12345%26request%3Dreq12345&meta.limit=20&meta.offset=0" + expected := map[string]interface{}{"filter": "parent=flow12345&request=req12345", "meta.limit": float64(20), "meta.offset": float64(0)} + actual := make(map[string]interface{}) + err := Unmarshal(&actual, query) + if err != nil { + t.Error(err) + } + for k, v := range actual { + if nv, ok := expected[k]; !ok || nv != v { + t.Errorf("Expected: %+v Actual: %+v", expected, actual) + } + } +}