diff --git a/go.sum b/go.sum index 370c571d..3ebc7ee5 100644 --- a/go.sum +++ b/go.sum @@ -295,7 +295,6 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxv github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= -github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= @@ -328,7 +327,6 @@ github.com/microcosm-cc/bluemonday v1.0.1/go.mod h1:hsXNsILzKxV+sX77C5b8FSuKF00v github.com/miekg/dns v1.1.15/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.27 h1:aEH/kqUzUxGJ/UHcEKdJY+ugH6WEzsEBBSPa8zuy1aM= github.com/miekg/dns v1.1.27/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= -github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-vnc v0.0.0-20150629162542-723ed9867aed/go.mod h1:3rdaFaCv4AyBgu5ALFM0+tSuHrBh6v692nyQe3ikrq0= github.com/mitchellh/hashstructure v1.0.0 h1:ZkRJX1CyOoTkar7p/mLS5TZU4nJ1Rn/F8u9dGS02Q3Y= @@ -370,14 +368,11 @@ github.com/nxadm/tail v1.4.4 h1:DQuhQpB1tVlglWS2hLQ5OV6B5r8aGxSrPc5Qo6uTN78= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/olekukonko/tablewriter v0.0.1/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= github.com/onsi/ginkgo v1.14.0 h1:2mOpI4JVVPBN+WQRa0WKH2eXR+Ey+uK4n7Zj0aYpIQA= github.com/onsi/ginkgo v1.14.0/go.mod h1:iSB4RoI2tjJc9BBv4NKIKWKya62Rps+oPG/Lv9klQyY= -github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/onsi/gomega v1.7.1 h1:K0jcRCwNQM3vFGh1ppMtDh/+7ApJrjldlX8fA0jDTLQ= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= @@ -416,14 +411,12 @@ github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXP github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM= github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= -github.com/prometheus/client_golang v1.1.0 h1:BQ53HtBmfOitExawJ6LokA4x8ov/z0SYYb0+HxJfRI8= github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g= github.com/prometheus/client_golang v1.7.0 h1:wCi7urQOGBsYcQROHqpUUX4ct84xp40t9R9JX0FuA/U= github.com/prometheus/client_golang v1.7.0/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 h1:gQz4mCbXsO+nc9n1hCxHcGA3Zx3Eo+UHZoInFGUIXNM= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -431,7 +424,6 @@ github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e/go.mod h1:daVV7q github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.6.0 h1:kRhiuYSXR3+uv2IbVbZhUxK5zVD/2pp3Gd2PpvPkpEo= github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc= github.com/prometheus/common v0.10.0 h1:RyRA7RzGXQZiW+tGMr7sxa85G1z0yOpM1qq5c8lNawc= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= @@ -441,7 +433,6 @@ github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= -github.com/prometheus/procfs v0.0.5 h1:3+auTFlqw+ZaQYJARz6ArODtkaIwtvBTx3N2NehQlL8= github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/prometheus/procfs v0.1.3 h1:F0+tqvhOksq22sc6iCHF5WGlWjdwj92p0udFh1VFBS8= @@ -451,7 +442,6 @@ github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqn github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= -github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sacloud/libsacloud v1.26.1/go.mod h1:79ZwATmHLIFZIMd7sxA3LwzVy/B77uj3LDoToVTxDoQ= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= @@ -476,7 +466,6 @@ github.com/shurcooL/notifications v0.0.0-20181007000457-627ab5aea122/go.mod h1:b github.com/shurcooL/octicon v0.0.0-20181028054416-fa4f57f9efb2/go.mod h1:eWdoE5JD4R5UVWDucdOPg1g2fqQRq78IQa9zlOV1vpQ= github.com/shurcooL/reactions v0.0.0-20181006231557-f2e0b4ca5b82/go.mod h1:TCR1lToEk4d2s07G3XGfz2QrgHXg4RJBvjrOozvoWfk= github.com/shurcooL/sanitized_anchor_name v0.0.0-20170918181015-86672fcb3f95/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= -github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/shurcooL/users v0.0.0-20180125191416-49c67e49c537/go.mod h1:QJTqeLYEDaXHZDBsXlPCDqdhQuJkuw4NOtaxYe3xii4= github.com/shurcooL/webdavfs v0.0.0-20170829043945-18c3829fa133/go.mod h1:hKmq5kWdCj2z2KEozexVbfEZIWiTjhE0+UjmZgPqehw= @@ -495,7 +484,6 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= @@ -701,7 +689,6 @@ golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20191216173652-a0e659d51361 h1:RIIXAeV6GvDBuADKumTODatUqANFZ+5BPMnzsy4hulY= golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa h1:5E4dL8+NgFOgjwbTKz+OOEGGhP+ectTmF842l6KjupQ= golang.org/x/tools v0.0.0-20200207183749-b753a1ba74fa/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= @@ -737,7 +724,6 @@ google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRn google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= -google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1 h1:aQktFqmDE2yjveXJlVIfslDFmFnUXSqG0i6KRcJAeMc= google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200423170343-7949de9c1215/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 h1:+kGHl1aib/qcwaRi1CbqBZ1rk19r85MNUf8HaBghugY= @@ -749,7 +735,6 @@ google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= -google.golang.org/protobuf v1.22.0 h1:cJv5/xdbk1NnMPR1VP9+HU6gupuG9MLBoH1r6RHZ2MY= google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= @@ -770,7 +755,6 @@ gopkg.in/ini.v1 v1.44.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ns1/ns1-go.v2 v2.0.0-20190730140822-b51389932cbc/go.mod h1:VV+3haRsgDiVLxyifmMBrBIuCWFBPYKbRssXB9z67Hw= gopkg.in/resty.v1 v1.9.1/go.mod h1:vo52Hzryw9PnPHcJfPsBiFW62XhNx5OczbV9y+IMpgc= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= -gopkg.in/square/go-jose.v2 v2.3.1 h1:SK5KegNXmKmqE342YYN2qPHEnUYeoMiXXl1poUlI+o4= gopkg.in/square/go-jose.v2 v2.3.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/square/go-jose.v2 v2.4.1 h1:H0TmLt7/KmzlrDOpa1F+zr0Tk90PbJYBfsVUmRLrf9Y= gopkg.in/square/go-jose.v2 v2.4.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= @@ -779,10 +763,8 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWD gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= diff --git a/store/cockroach/cockroach.go b/store/cockroach/cockroach.go index 66cbb4f3..5cbedb60 100644 --- a/store/cockroach/cockroach.go +++ b/store/cockroach/cockroach.go @@ -27,7 +27,7 @@ var ( re = regexp.MustCompile("[^a-zA-Z0-9]+") statements = map[string]string{ - "list": "SELECT key, value, metadata, expiry FROM %s.%s;", + "list": "SELECT key, value, metadata, expiry FROM %s.%s WHERE key LIKE $1 ORDER BY key DESC LIMIT $2 OFFSET $3;", "read": "SELECT key, value, metadata, expiry FROM %s.%s WHERE key = $1;", "readMany": "SELECT key, value, metadata, expiry FROM %s.%s WHERE key LIKE $1;", "readOffset": "SELECT key, value, metadata, expiry FROM %s.%s WHERE key LIKE $1 ORDER BY key DESC LIMIT $2 OFFSET $3;", @@ -213,6 +213,23 @@ func (s *sqlStore) List(opts ...store.ListOption) ([]string, error) { if err := s.createDB(options.Database, options.Table); err != nil { return nil, err } + limit := sql.NullInt32{} + offset := 0 + pattern := "%" + if options.Prefix != "" || options.Suffix != "" { + if options.Prefix != "" { + pattern = options.Prefix + pattern + } + if options.Suffix != "" { + pattern = pattern + options.Suffix + } + } + if options.Offset > 0 { + offset = int(options.Offset) + } + if options.Limit > 0 { + limit = sql.NullInt32{Int32: int32(options.Limit), Valid: true} + } st, err := s.prepare(options.Database, options.Table, "list") if err != nil { @@ -220,7 +237,7 @@ func (s *sqlStore) List(opts ...store.ListOption) ([]string, error) { } defer st.Close() - rows, err := st.Query() + rows, err := st.Query(pattern, limit, offset) if err != nil { if err == sql.ErrNoRows { return nil, nil @@ -228,8 +245,55 @@ func (s *sqlStore) List(opts ...store.ListOption) ([]string, error) { return nil, err } defer rows.Close() - var keys []string + records, err := s.rowsToRecords(rows) + if err != nil { + return nil, err + } + for _, k := range records { + keys = append(keys, k.Key) + } + rowErr := rows.Close() + if rowErr != nil { + // transaction rollback or something + return keys, rowErr + } + if err := rows.Err(); err != nil { + return keys, err + } + return keys, nil +} + +// rowToRecord converts from sql.Row to a store.Record. If the record has expired it will issue a delete in a separate goroutine +func (s *sqlStore) rowToRecord(row *sql.Row) (*store.Record, error) { + var timehelper pq.NullTime + record := &store.Record{} + metadata := make(Metadata) + + if err := row.Scan(&record.Key, &record.Value, &metadata, &timehelper); err != nil { + if err == sql.ErrNoRows { + return record, store.ErrNotFound + } + return nil, err + } + + // set the metadata + record.Metadata = toMetadata(&metadata) + if timehelper.Valid { + if timehelper.Time.Before(time.Now()) { + // record has expired + go s.Delete(record.Key) + return nil, store.ErrNotFound + } + record.Expiry = time.Until(timehelper.Time) + + } + return record, nil +} + +// rowsToRecords converts from sql.Rows to []*store.Record. If a record has expired it will issue a delete in a separate goroutine +func (s *sqlStore) rowsToRecords(rows *sql.Rows) ([]*store.Record, error) { + var records []*store.Record var timehelper pq.NullTime for rows.Next() { @@ -237,7 +301,7 @@ func (s *sqlStore) List(opts ...store.ListOption) ([]string, error) { metadata := make(Metadata) if err := rows.Scan(&record.Key, &record.Value, &metadata, &timehelper); err != nil { - return keys, err + return records, err } // set the metadata @@ -249,22 +313,13 @@ func (s *sqlStore) List(opts ...store.ListOption) ([]string, error) { go s.Delete(record.Key) } else { record.Expiry = time.Until(timehelper.Time) - keys = append(keys, record.Key) + records = append(records, record) } } else { - keys = append(keys, record.Key) + records = append(records, record) } - } - rowErr := rows.Close() - if rowErr != nil { - // transaction rollback or something - return keys, rowErr - } - if err := rows.Err(); err != nil { - return keys, err - } - return keys, nil + return records, nil } // Read a single key @@ -283,9 +338,6 @@ func (s *sqlStore) Read(key string, opts ...store.ReadOption) ([]*store.Record, return s.read(key, options) } - var records []*store.Record - var timehelper pq.NullTime - st, err := s.prepare(options.Database, options.Table, "read") if err != nil { return nil, err @@ -293,32 +345,12 @@ func (s *sqlStore) Read(key string, opts ...store.ReadOption) ([]*store.Record, defer st.Close() row := st.QueryRow(key) - record := &store.Record{} - metadata := make(Metadata) - - if err := row.Scan(&record.Key, &record.Value, &metadata, &timehelper); err != nil { - if err == sql.ErrNoRows { - return records, store.ErrNotFound - } - return records, err + record, err := s.rowToRecord(row) + if err != nil { + return nil, err } - - // set the metadata - record.Metadata = toMetadata(&metadata) - - if timehelper.Valid { - if timehelper.Time.Before(time.Now()) { - // record has expired - go s.Delete(key) - return records, store.ErrNotFound - } - record.Expiry = time.Until(timehelper.Time) - records = append(records, record) - } else { - records = append(records, record) - } - - return records, nil + var records []*store.Record + return append(records, record), nil } // Read Many records @@ -361,31 +393,9 @@ func (s *sqlStore) read(key string, options store.ReadOptions) ([]*store.Record, defer rows.Close() - var records []*store.Record - var timehelper pq.NullTime - - for rows.Next() { - record := &store.Record{} - metadata := make(Metadata) - - if err := rows.Scan(&record.Key, &record.Value, &metadata, &timehelper); err != nil { - return records, err - } - - // set the metadata - record.Metadata = toMetadata(&metadata) - - if timehelper.Valid { - if timehelper.Time.Before(time.Now()) { - // record has expired - go s.Delete(record.Key) - } else { - record.Expiry = time.Until(timehelper.Time) - records = append(records, record) - } - } else { - records = append(records, record) - } + records, err := s.rowsToRecords(rows) + if err != nil { + return nil, err } rowErr := rows.Close() if rowErr != nil { diff --git a/store/file/file.go b/store/file/file.go index 2b171f1d..64433ee3 100644 --- a/store/file/file.go +++ b/store/file/file.go @@ -153,13 +153,18 @@ func (m *fileStore) list(db *bolt.DB, limit, offset uint) []string { if limit != 0 || offset != 0 { sort.Slice(allKeys, func(i, j int) bool { return allKeys[i] < allKeys[j] }) - min := func(i, j uint) uint { - if i < j { - return i + end := len(allKeys) + if limit > 0 { + calcLimit := int(offset + limit) + if calcLimit < end { + end = calcLimit } - return j } - return allKeys[offset:min(limit, uint(len(allKeys)))] + + if int(offset) >= end { + return nil + } + return allKeys[offset:end] } return allKeys diff --git a/store/memory/memory.go b/store/memory/memory.go index 63947565..38aae279 100644 --- a/store/memory/memory.go +++ b/store/memory/memory.go @@ -136,13 +136,19 @@ func (m *memoryStore) list(prefix string, limit, offset uint) []string { if limit != 0 || offset != 0 { sort.Slice(allKeys, func(i, j int) bool { return allKeys[i] < allKeys[j] }) - min := func(i, j uint) uint { - if i < j { - return i + sort.Slice(allKeys, func(i, j int) bool { return allKeys[i] < allKeys[j] }) + end := len(allKeys) + if limit > 0 { + calcLimit := int(offset + limit) + if calcLimit < end { + end = calcLimit } - return j } - return allKeys[offset:min(limit, uint(len(allKeys)))] + + if int(offset) >= end { + return nil + } + return allKeys[offset:end] } return allKeys diff --git a/store/test/store_test.go b/store/test/store_test.go new file mode 100644 index 00000000..5d3d56a3 --- /dev/null +++ b/store/test/store_test.go @@ -0,0 +1,471 @@ +// Package test provides a way to run tests against all the various implementations of the Store interface. +// It can't live in the store package itself because of circular import issues +package test + +import ( + "fmt" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/unistack-org/micro/v3/store/cache" + + "github.com/unistack-org/micro/v3/store/memory" + + "github.com/kr/pretty" + "github.com/unistack-org/micro/v3/store/cockroach" + + "github.com/unistack-org/micro/v3/store" + + "github.com/unistack-org/micro/v3/store/file" + + "github.com/davecgh/go-spew/spew" +) + +func fileStoreCleanup(db string, s store.Store) { + s.Close() + dir := filepath.Join(file.DefaultDir, db+"/") + os.RemoveAll(dir) +} + +func cockroachStoreCleanup(db string, s store.Store) { + keys, _ := s.List() + for _, k := range keys { + s.Delete(k) + } + s.Close() +} + +func memoryCleanup(db string, s store.Store) { + s.Close() +} + +func cacheCleanup(db string, s store.Store) { + s.Close() +} + +func TestStoreReInit(t *testing.T) { + tcs := []struct { + name string + s store.Store + cleanup func(db string, s store.Store) + }{ + {name: "file", s: file.NewStore(store.Table("aaa")), cleanup: fileStoreCleanup}, + {name: "cockroach", s: cockroach.NewStore(store.Table("aaa")), cleanup: cockroachStoreCleanup}, + {name: "memory", s: memory.NewStore(store.Table("aaa")), cleanup: memoryCleanup}, + {name: "cache", s: cache.NewStore(memory.NewStore(store.Table("aaa"))), cleanup: cacheCleanup}, + } + for _, tc := range tcs { + t.Run(tc.name, func(t *testing.T) { + defer tc.cleanup(file.DefaultDatabase, tc.s) + tc.s.Init(store.Table("bbb")) + if tc.s.Options().Table != "bbb" { + t.Error("Init didn't reinitialise the store") + } + }) + } +} + +func TestStoreBasic(t *testing.T) { + tcs := []struct { + name string + s store.Store + cleanup func(db string, s store.Store) + }{ + {name: "file", s: file.NewStore(), cleanup: fileStoreCleanup}, + {name: "cockroach", s: cockroach.NewStore(), cleanup: cockroachStoreCleanup}, + {name: "memory", s: memory.NewStore(), cleanup: memoryCleanup}, + {name: "cache", s: cache.NewStore(memory.NewStore()), cleanup: cacheCleanup}, + } + for _, tc := range tcs { + t.Run(tc.name, func(t *testing.T) { + defer tc.cleanup(file.DefaultDatabase, tc.s) + runStoreTest(tc.s, t) + }) + } + +} + +func TestStoreTable(t *testing.T) { + tcs := []struct { + name string + s store.Store + cleanup func(db string, s store.Store) + }{ + {name: "file", s: file.NewStore(store.Table("testTable")), cleanup: fileStoreCleanup}, + {name: "cockroach", s: cockroach.NewStore(store.Table("testTable")), cleanup: cockroachStoreCleanup}, + {name: "memory", s: memory.NewStore(store.Table("testTable")), cleanup: memoryCleanup}, + {name: "cache", s: cache.NewStore(memory.NewStore(store.Table("testTable"))), cleanup: cacheCleanup}, + } + for _, tc := range tcs { + t.Run(tc.name, func(t *testing.T) { + defer tc.cleanup(file.DefaultDatabase, tc.s) + runStoreTest(tc.s, t) + }) + } +} + +func TestStoreDatabase(t *testing.T) { + tcs := []struct { + name string + s store.Store + cleanup func(db string, s store.Store) + }{ + {name: "file", s: file.NewStore(store.Database("testdb")), cleanup: fileStoreCleanup}, + {name: "cockroach", s: cockroach.NewStore(store.Database("testdb")), cleanup: cockroachStoreCleanup}, + {name: "memory", s: memory.NewStore(store.Database("testdb")), cleanup: memoryCleanup}, + {name: "cache", s: cache.NewStore(memory.NewStore(store.Database("testdb"))), cleanup: cacheCleanup}, + } + for _, tc := range tcs { + t.Run(tc.name, func(t *testing.T) { + defer tc.cleanup("testdb", tc.s) + runStoreTest(tc.s, t) + }) + } +} + +func TestStoreDatabaseTable(t *testing.T) { + tcs := []struct { + name string + s store.Store + cleanup func(db string, s store.Store) + }{ + {name: "file", s: file.NewStore(store.Database("testdb"), store.Table("testTable")), cleanup: fileStoreCleanup}, + {name: "cockroach", s: cockroach.NewStore(store.Database("testdb"), store.Table("testTable")), cleanup: cockroachStoreCleanup}, + {name: "memory", s: memory.NewStore(store.Database("testdb"), store.Table("testTable")), cleanup: memoryCleanup}, + {name: "cache", s: cache.NewStore(memory.NewStore(store.Database("testdb"), store.Table("testTable"))), cleanup: cacheCleanup}, + } + for _, tc := range tcs { + t.Run(tc.name, func(t *testing.T) { + defer tc.cleanup("testdb", tc.s) + runStoreTest(tc.s, t) + }) + } +} + +func runStoreTest(s store.Store, t *testing.T) { + if len(os.Getenv("IN_TRAVIS_CI")) == 0 { + t.Logf("Options %s %v\n", s.String(), s.Options()) + } + + expiryTests(s, t) + suffixPrefixExpiryTests(s, t) + readTests(s, t) + listTests(s, t) + +} + +func readTests(s store.Store, t *testing.T) { + // Test Table, Suffix and WriteOptions + if err := s.Write(&store.Record{ + Key: "foofoobarbar", + Value: []byte("something"), + }, store.WriteTTL(time.Millisecond*100)); err != nil { + t.Error(err) + } + if err := s.Write(&store.Record{ + Key: "foofoo", + Value: []byte("something"), + }, store.WriteExpiry(time.Now().Add(time.Millisecond*100))); err != nil { + t.Error(err) + } + if err := s.Write(&store.Record{ + Key: "barbar", + Value: []byte("something"), + // TTL has higher precedence than expiry + }, store.WriteExpiry(time.Now().Add(time.Hour)), store.WriteTTL(time.Millisecond*100)); err != nil { + t.Error(err) + } + + if results, err := s.Read("foo", store.ReadPrefix(), store.ReadSuffix()); err != nil { + t.Error(err) + } else { + if len(results) != 1 { + t.Errorf("Expected 1 results, got %d: %# v", len(results), spew.Sdump(results)) + } + } + + time.Sleep(time.Millisecond * 100) + + if results, err := s.List(); err != nil { + t.Fatalf("List failed: %s", err) + } else { + if len(results) != 0 { + t.Fatalf("Expiry options were not effective, results :%v", spew.Sdump(results)) + } + } + + // write the following records + for i := 0; i < 10; i++ { + s.Write(&store.Record{ + Key: fmt.Sprintf("a%d", i), + Value: []byte{}, + }) + } + + // read back a few records + if results, err := s.Read("a", store.ReadLimit(5), store.ReadPrefix()); err != nil { + t.Error(err) + } else { + if len(results) != 5 { + t.Fatal("Expected 5 results, got ", len(results)) + } + if !strings.HasPrefix(results[0].Key, "a") { + t.Fatalf("Expected a prefix, got %s", results[0].Key) + } + } + + // read the rest back + if results, err := s.Read("a", store.ReadLimit(30), store.ReadOffset(5), store.ReadPrefix()); err != nil { + t.Fatal(err) + } else { + if len(results) != 5 { + t.Fatal("Expected 5 results, got ", len(results)) + } + } +} + +func listTests(s store.Store, t *testing.T) { + for i := 0; i < 10; i++ { + s.Write(&store.Record{Key: fmt.Sprintf("List%d", i), Value: []byte("bar")}) + } + + recs, err := s.List(store.ListPrefix("List")) + if err != nil { + t.Fatalf("Error listing records %s", err) + } + if len(recs) != 10 { + t.Fatalf("Expected 10 records, received %d", len(recs)) + } + + recs, err = s.List(store.ListPrefix("List"), store.ListLimit(5)) + if err != nil { + t.Fatalf("Error listing records %s", err) + } + if len(recs) != 5 { + t.Fatalf("Expected 5 records, received %d", len(recs)) + } + + recs, err = s.List(store.ListPrefix("List"), store.ListOffset(6)) + if err != nil { + t.Fatalf("Error listing records %s", err) + } + if len(recs) != 4 { + t.Fatalf("Expected 4 records, received %d %+v", len(recs), recs) + } + + recs, err = s.List(store.ListPrefix("List"), store.ListOffset(6), store.ListLimit(2)) + if err != nil { + t.Fatalf("Error listing records %s", err) + } + if len(recs) != 2 { + t.Fatalf("Expected 2 records, received %d %+v", len(recs), recs) + } + +} + +func expiryTests(s store.Store, t *testing.T) { + // Read and Write an expiring Record + if err := s.Write(&store.Record{ + Key: "Hello", + Value: []byte("World"), + Expiry: time.Millisecond * 150, + }); err != nil { + t.Error(err) + } + + if r, err := s.Read("Hello"); err != nil { + t.Fatal(err) + } else { + if len(r) != 1 { + t.Error("Read returned multiple records") + } + if r[0].Key != "Hello" { + t.Errorf("Expected %s, got %s", "Hello", r[0].Key) + } + if string(r[0].Value) != "World" { + t.Errorf("Expected %s, got %s", "World", r[0].Value) + } + } + + // wait for expiry + time.Sleep(time.Millisecond * 200) + + if _, err := s.Read("Hello"); err != store.ErrNotFound { + t.Errorf("Expected %# v, got %# v", store.ErrNotFound, err) + } + + // exercise the different ways to write an expiry, record.expiry + s.Write(&store.Record{Key: "aaa", Value: []byte("bbb"), Expiry: 1 * time.Second}) + s.Write(&store.Record{Key: "aaaa", Value: []byte("bbb"), Expiry: 1 * time.Second}) + s.Write(&store.Record{Key: "aaaaa", Value: []byte("bbb"), Expiry: 1 * time.Second}) + results, err := s.Read("a", store.ReadPrefix()) + if err != nil { + t.Error(err) + } + if len(results) != 3 { + t.Fatal("Results should have returned 3 records") + } + time.Sleep(1 * time.Second) + results, err = s.Read("a", store.ReadPrefix()) + if err != nil { + t.Error(err) + } + if len(results) != 0 { + t.Fatal("Results should have returned 0 records") + } + + // exercise the different ways to write an expiry, WriteExpiry + s.Write(&store.Record{Key: "bbb", Value: []byte("bbb")}, store.WriteExpiry(time.Now().Add(1*time.Second))) + s.Write(&store.Record{Key: "bbbb", Value: []byte("bbb")}, store.WriteExpiry(time.Now().Add(1*time.Second))) + s.Write(&store.Record{Key: "bbbbb", Value: []byte("bbb")}, store.WriteExpiry(time.Now().Add(1*time.Second))) + results, err = s.Read("b", store.ReadPrefix()) + if err != nil { + t.Error(err) + } + if len(results) != 3 { + t.Fatalf("Results should have returned 3 records. Received %d", len(results)) + } + time.Sleep(1 * time.Second) + results, err = s.Read("b", store.ReadPrefix()) + if err != nil { + t.Error(err) + } + if len(results) != 0 { + t.Fatalf("Results should have returned 0 records. Received %d", len(results)) + } + + // exercise the different ways to write an expiry, WriteTTL + s.Write(&store.Record{Key: "ccc", Value: []byte("bbb")}, store.WriteTTL(1*time.Second)) + s.Write(&store.Record{Key: "cccc", Value: []byte("bbb")}, store.WriteTTL(1*time.Second)) + s.Write(&store.Record{Key: "ccccc", Value: []byte("bbb")}, store.WriteTTL(1*time.Second)) + results, err = s.Read("c", store.ReadPrefix()) + if err != nil { + t.Error(err) + } + if len(results) != 3 { + t.Fatalf("Results should have returned 3 records. Received %d", len(results)) + } + time.Sleep(1 * time.Second) + results, err = s.Read("c", store.ReadPrefix()) + if err != nil { + t.Error(err) + } + if len(results) != 0 { + t.Fatalf("Results should have returned 0 records. Received %d", len(results)) + } +} + +func suffixPrefixExpiryTests(s store.Store, t *testing.T) { + // Write 3 records with various expiry and get with Prefix + records := []*store.Record{ + &store.Record{ + Key: "foo", + Value: []byte("foofoo"), + }, + &store.Record{ + Key: "foobar", + Value: []byte("foobarfoobar"), + Expiry: time.Millisecond * 100, + }, + } + + for _, r := range records { + if err := s.Write(r); err != nil { + t.Errorf("Couldn't write k: %s, v: %# v (%s)", r.Key, pretty.Formatter(r.Value), err) + } + } + + if results, err := s.Read("foo", store.ReadPrefix()); err != nil { + t.Errorf("Couldn't read all \"foo\" keys, got %#v (%s)", spew.Sdump(results), err) + } else { + if len(results) != 2 { + t.Errorf("Expected 2 items, got %d", len(results)) + } + } + + // wait for the expiry + time.Sleep(time.Millisecond * 200) + + if results, err := s.Read("foo", store.ReadPrefix()); err != nil { + t.Errorf("Couldn't read all \"foo\" keys, got %# v (%s)", spew.Sdump(results), err) + } else if len(results) != 1 { + t.Errorf("Expected 1 item, got %d", len(results)) + } + + if err := s.Delete("foo"); err != nil { + t.Errorf("Delete failed (%v)", err) + } + + if results, err := s.Read("foo"); err != store.ErrNotFound { + t.Errorf("Expected read failure read all \"foo\" keys, got %# v (%s)", spew.Sdump(results), err) + } else { + if len(results) != 0 { + t.Errorf("Expected 0 items, got %d (%# v)", len(results), spew.Sdump(results)) + } + } + + // Write 3 records with various expiry and get with Suffix + records = []*store.Record{ + &store.Record{ + Key: "foo", + Value: []byte("foofoo"), + }, + &store.Record{ + Key: "barfoo", + Value: []byte("barfoobarfoo"), + + Expiry: time.Millisecond * 100, + }, + &store.Record{ + Key: "bazbarfoo", + Value: []byte("bazbarfoobazbarfoo"), + Expiry: 2 * time.Millisecond * 100, + }, + } + for _, r := range records { + if err := s.Write(r); err != nil { + t.Errorf("Couldn't write k: %s, v: %# v (%s)", r.Key, pretty.Formatter(r.Value), err) + } + } + if results, err := s.Read("foo", store.ReadSuffix()); err != nil { + t.Errorf("Couldn't read all \"foo\" keys, got %# v (%s)", spew.Sdump(results), err) + } else { + if len(results) != 3 { + t.Errorf("Expected 3 items, got %d", len(results)) + //t.Logf("Table test: %v\n", spew.Sdump(results)) + } + + } + time.Sleep(time.Millisecond * 100) + if results, err := s.Read("foo", store.ReadSuffix()); err != nil { + t.Errorf("Couldn't read all \"foo\" keys, got %# v (%s)", spew.Sdump(results), err) + } else { + if len(results) != 2 { + t.Errorf("Expected 2 items, got %d", len(results)) + //t.Logf("Table test: %v\n", spew.Sdump(results)) + } + + } + time.Sleep(time.Millisecond * 100) + if results, err := s.Read("foo", store.ReadSuffix()); err != nil { + t.Errorf("Couldn't read all \"foo\" keys, got %# v (%s)", spew.Sdump(results), err) + } else { + if len(results) != 1 { + t.Errorf("Expected 1 item, got %d", len(results)) + // t.Logf("Table test: %# v\n", spew.Sdump(results)) + } + } + if err := s.Delete("foo"); err != nil { + t.Errorf("Delete failed (%v)", err) + } + if results, err := s.Read("foo", store.ReadSuffix()); err != nil { + t.Errorf("Couldn't read all \"foo\" keys, got %# v (%s)", spew.Sdump(results), err) + } else { + if len(results) != 0 { + t.Errorf("Expected 0 items, got %d (%# v)", len(results), spew.Sdump(results)) + } + } +}