Unify the store tests (#1952)

Add more tests for store

.github/workflows/pr.yml

@@ -25,4 +25,7 @@ jobs:
       id: tests
       env:
         IN_TRAVIS_CI: yes
-      run: go test -v ./...
+      run: |
+        wget -qO- https://binaries.cockroachdb.com/cockroach-v20.1.4.linux-amd64.tgz | tar  xvz
+        cockroach-v20.1.4.linux-amd64/cockroach start-single-node --insecure &
+        go test -v ./...

go.sum

@@ -276,7 +276,6 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxv
 github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s=
 github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
 github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc=
-github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs=
 github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI=
@@ -306,7 +305,6 @@ github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5
 github.com/miekg/dns v1.1.15/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
 github.com/miekg/dns v1.1.27 h1:aEH/kqUzUxGJ/UHcEKdJY+ugH6WEzsEBBSPa8zuy1aM=
 github.com/miekg/dns v1.1.27/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
-github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
 github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
 github.com/mitchellh/go-vnc v0.0.0-20150629162542-723ed9867aed/go.mod h1:3rdaFaCv4AyBgu5ALFM0+tSuHrBh6v692nyQe3ikrq0=
 github.com/mitchellh/hashstructure v1.0.0 h1:ZkRJX1CyOoTkar7p/mLS5TZU4nJ1Rn/F8u9dGS02Q3Y=
@@ -346,11 +344,9 @@ github.com/nrdcg/goinwx v0.6.1/go.mod h1:XPiut7enlbEdntAqalBIqcYcTEVhpv/dKWgDCX2
 github.com/nrdcg/namesilo v0.2.1/go.mod h1:lwMvfQTyYq+BbjJd30ylEG4GPSS6PII0Tia4rRpRiyw=
 github.com/olekukonko/tablewriter v0.0.1/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo=
 github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
-github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs=
 github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE=
 github.com/onsi/ginkgo v1.12.0 h1:Iw5WCbBcaAAd0fpRb1c9r5YCylv4XDoCSigm1zLevwU=
 github.com/onsi/ginkgo v1.12.0/go.mod h1:oUhWkIvk5aDxtKvDDuw8gItl8pKl42LzjC9KZE0HfGg=
-github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU=
 github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY=
 github.com/onsi/gomega v1.7.1 h1:K0jcRCwNQM3vFGh1ppMtDh/+7ApJrjldlX8fA0jDTLQ=
 github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
@@ -385,21 +381,18 @@ github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXP
 github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM=
 github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs=
 github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo=
-github.com/prometheus/client_golang v1.1.0 h1:BQ53HtBmfOitExawJ6LokA4x8ov/z0SYYb0+HxJfRI8=
 github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g=
 github.com/prometheus/client_golang v1.7.0 h1:wCi7urQOGBsYcQROHqpUUX4ct84xp40t9R9JX0FuA/U=
 github.com/prometheus/client_golang v1.7.0/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M=
 github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo=
 github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
-github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 h1:gQz4mCbXsO+nc9n1hCxHcGA3Zx3Eo+UHZoInFGUIXNM=
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M=
 github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro=
 github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
 github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4=
-github.com/prometheus/common v0.6.0 h1:kRhiuYSXR3+uv2IbVbZhUxK5zVD/2pp3Gd2PpvPkpEo=
 github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc=
 github.com/prometheus/common v0.10.0 h1:RyRA7RzGXQZiW+tGMr7sxa85G1z0yOpM1qq5c8lNawc=
 github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo=
@@ -408,7 +401,6 @@ github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R
 github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk=
 github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA=
 github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ=
-github.com/prometheus/procfs v0.0.5 h1:3+auTFlqw+ZaQYJARz6ArODtkaIwtvBTx3N2NehQlL8=
 github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ=
 github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A=
 github.com/prometheus/procfs v0.1.3 h1:F0+tqvhOksq22sc6iCHF5WGlWjdwj92p0udFh1VFBS8=
@@ -417,11 +409,9 @@ github.com/rainycape/memcache v0.0.0-20150622160815-1031fa0ce2f2/go.mod h1:7tZKc
 github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
 github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
-github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q=
 github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
 github.com/sacloud/libsacloud v1.26.1/go.mod h1:79ZwATmHLIFZIMd7sxA3LwzVy/B77uj3LDoToVTxDoQ=
 github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
-github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo=
 github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc=
 github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
 github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q=
@@ -436,7 +426,6 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+
 github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
 github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs=
 github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI=
-github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk=
 github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4=
 github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4=
 github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA=
@@ -610,7 +599,6 @@ golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtn
 golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo=
 golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
-golang.org/x/tools v0.0.0-20191216173652-a0e659d51361 h1:RIIXAeV6GvDBuADKumTODatUqANFZ+5BPMnzsy4hulY=
 golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
 golang.org/x/tools v0.0.0-20200117065230-39095c1d176c h1:FodBYPZKH5tAN2O60HlglMwXGAeV/4k+NKbli79M/2c=
 golang.org/x/tools v0.0.0-20200117065230-39095c1d176c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28=
@@ -639,7 +627,6 @@ google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRn
 google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
 google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc=
 google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8=
-google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1 h1:aQktFqmDE2yjveXJlVIfslDFmFnUXSqG0i6KRcJAeMc=
 google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc=
 google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 h1:+kGHl1aib/qcwaRi1CbqBZ1rk19r85MNUf8HaBghugY=
 google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo=
@@ -650,7 +637,6 @@ google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ
 google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM=
 google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE=
 google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo=
-google.golang.org/protobuf v1.22.0 h1:cJv5/xdbk1NnMPR1VP9+HU6gupuG9MLBoH1r6RHZ2MY=
 google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
 google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
 google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU=
@@ -671,7 +657,6 @@ gopkg.in/ini.v1 v1.44.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
 gopkg.in/ns1/ns1-go.v2 v2.0.0-20190730140822-b51389932cbc/go.mod h1:VV+3haRsgDiVLxyifmMBrBIuCWFBPYKbRssXB9z67Hw=
 gopkg.in/resty.v1 v1.9.1/go.mod h1:vo52Hzryw9PnPHcJfPsBiFW62XhNx5OczbV9y+IMpgc=
 gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
-gopkg.in/square/go-jose.v2 v2.3.1 h1:SK5KegNXmKmqE342YYN2qPHEnUYeoMiXXl1poUlI+o4=
 gopkg.in/square/go-jose.v2 v2.3.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI=
 gopkg.in/square/go-jose.v2 v2.4.1 h1:H0TmLt7/KmzlrDOpa1F+zr0Tk90PbJYBfsVUmRLrf9Y=
 gopkg.in/square/go-jose.v2 v2.4.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI=
@@ -680,7 +665,6 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWD
 gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
 gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
-gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I=
 gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
 gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10=

store/cockroach/cockroach.go

@@ -27,7 +27,7 @@ var (
 	re = regexp.MustCompile("[^a-zA-Z0-9]+")
 
 	statements = map[string]string{
-		"list":       "SELECT key, value, metadata, expiry FROM %s.%s;",
+		"list":       "SELECT key, value, metadata, expiry FROM %s.%s WHERE key LIKE $1 ORDER BY key DESC LIMIT $2 OFFSET $3;",
 		"read":       "SELECT key, value, metadata, expiry FROM %s.%s WHERE key = $1;",
 		"readMany":   "SELECT key, value, metadata, expiry FROM %s.%s WHERE key LIKE $1;",
 		"readOffset": "SELECT key, value, metadata, expiry FROM %s.%s WHERE key LIKE $1 ORDER BY key DESC LIMIT $2 OFFSET $3;",
@@ -213,6 +213,23 @@ func (s *sqlStore) List(opts ...store.ListOption) ([]string, error) {
 	if err := s.createDB(options.Database, options.Table); err != nil {
 		return nil, err
 	}
+	limit := sql.NullInt32{}
+	offset := 0
+	pattern := "%"
+	if options.Prefix != "" || options.Suffix != "" {
+		if options.Prefix != "" {
+			pattern = options.Prefix + pattern
+		}
+		if options.Suffix != "" {
+			pattern = pattern + options.Suffix
+		}
+	}
+	if options.Offset > 0 {
+		offset = int(options.Offset)
+	}
+	if options.Limit > 0 {
+		limit = sql.NullInt32{Int32: int32(options.Limit), Valid: true}
+	}
 
 	st, err := s.prepare(options.Database, options.Table, "list")
 	if err != nil {
@@ -220,7 +237,7 @@ func (s *sqlStore) List(opts ...store.ListOption) ([]string, error) {
 	}
 	defer st.Close()
 
-	rows, err := st.Query()
+	rows, err := st.Query(pattern, limit, offset)
 	if err != nil {
 		if err == sql.ErrNoRows {
 			return nil, nil
@@ -228,8 +245,55 @@ func (s *sqlStore) List(opts ...store.ListOption) ([]string, error) {
 		return nil, err
 	}
 	defer rows.Close()
-
 	var keys []string
+	records, err := s.rowsToRecords(rows)
+	if err != nil {
+		return nil, err
+	}
+	for _, k := range records {
+		keys = append(keys, k.Key)
+	}
+	rowErr := rows.Close()
+	if rowErr != nil {
+		// transaction rollback or something
+		return keys, rowErr
+	}
+	if err := rows.Err(); err != nil {
+		return keys, err
+	}
+	return keys, nil
+}
+
+// rowToRecord converts from sql.Row to a store.Record. If the record has expired it will issue a delete in a separate goroutine
+func (s *sqlStore) rowToRecord(row *sql.Row) (*store.Record, error) {
+	var timehelper pq.NullTime
+	record := &store.Record{}
+	metadata := make(Metadata)
+
+	if err := row.Scan(&record.Key, &record.Value, &metadata, &timehelper); err != nil {
+		if err == sql.ErrNoRows {
+			return record, store.ErrNotFound
+		}
+		return nil, err
+	}
+
+	// set the metadata
+	record.Metadata = toMetadata(&metadata)
+	if timehelper.Valid {
+		if timehelper.Time.Before(time.Now()) {
+			// record has expired
+			go s.Delete(record.Key)
+			return nil, store.ErrNotFound
+		}
+		record.Expiry = time.Until(timehelper.Time)
+
+	}
+	return record, nil
+}
+
+// rowsToRecords converts from sql.Rows to  []*store.Record. If a record has expired it will issue a delete in a separate goroutine
+func (s *sqlStore) rowsToRecords(rows *sql.Rows) ([]*store.Record, error) {
+	var records []*store.Record
 	var timehelper pq.NullTime
 
 	for rows.Next() {
@@ -237,7 +301,7 @@ func (s *sqlStore) List(opts ...store.ListOption) ([]string, error) {
 		metadata := make(Metadata)
 
 		if err := rows.Scan(&record.Key, &record.Value, &metadata, &timehelper); err != nil {
-			return keys, err
+			return records, err
 		}
 
 		// set the metadata
@@ -249,22 +313,13 @@ func (s *sqlStore) List(opts ...store.ListOption) ([]string, error) {
 				go s.Delete(record.Key)
 			} else {
 				record.Expiry = time.Until(timehelper.Time)
-				keys = append(keys, record.Key)
+				records = append(records, record)
 			}
 		} else {
-			keys = append(keys, record.Key)
+			records = append(records, record)
 		}
-
 	}
-	rowErr := rows.Close()
+	return records, nil
-	if rowErr != nil {
-		// transaction rollback or something
-		return keys, rowErr
-	}
-	if err := rows.Err(); err != nil {
-		return keys, err
-	}
-	return keys, nil
 }
 
 // Read a single key
@@ -283,9 +338,6 @@ func (s *sqlStore) Read(key string, opts ...store.ReadOption) ([]*store.Record,
 		return s.read(key, options)
 	}
 
-	var records []*store.Record
-	var timehelper pq.NullTime
-
 	st, err := s.prepare(options.Database, options.Table, "read")
 	if err != nil {
 		return nil, err
@@ -293,32 +345,12 @@ func (s *sqlStore) Read(key string, opts ...store.ReadOption) ([]*store.Record,
 	defer st.Close()
 
 	row := st.QueryRow(key)
-	record := &store.Record{}
+	record, err := s.rowToRecord(row)
-	metadata := make(Metadata)
+	if err != nil {
-
+		return nil, err
-	if err := row.Scan(&record.Key, &record.Value, &metadata, &timehelper); err != nil {
-		if err == sql.ErrNoRows {
-			return records, store.ErrNotFound
 	}
-		return records, err
+	var records []*store.Record
-	}
+	return append(records, record), nil
-
-	// set the metadata
-	record.Metadata = toMetadata(&metadata)
-
-	if timehelper.Valid {
-		if timehelper.Time.Before(time.Now()) {
-			// record has expired
-			go s.Delete(key)
-			return records, store.ErrNotFound
-		}
-		record.Expiry = time.Until(timehelper.Time)
-		records = append(records, record)
-	} else {
-		records = append(records, record)
-	}
-
-	return records, nil
 }
 
 // Read Many records
@@ -361,31 +393,9 @@ func (s *sqlStore) read(key string, options store.ReadOptions) ([]*store.Record,
 
 	defer rows.Close()
 
-	var records []*store.Record
+	records, err := s.rowsToRecords(rows)
-	var timehelper pq.NullTime
+	if err != nil {
-
+		return nil, err
-	for rows.Next() {
-		record := &store.Record{}
-		metadata := make(Metadata)
-
-		if err := rows.Scan(&record.Key, &record.Value, &metadata, &timehelper); err != nil {
-			return records, err
-		}
-
-		// set the metadata
-		record.Metadata = toMetadata(&metadata)
-
-		if timehelper.Valid {
-			if timehelper.Time.Before(time.Now()) {
-				// record has expired
-				go s.Delete(record.Key)
-			} else {
-				record.Expiry = time.Until(timehelper.Time)
-				records = append(records, record)
-			}
-		} else {
-			records = append(records, record)
-		}
 	}
 	rowErr := rows.Close()
 	if rowErr != nil {

store/cockroach/cockroach_test.go

@@ -1,163 +0,0 @@
-package cockroach
-
-import (
-	"database/sql"
-	"fmt"
-	"testing"
-	"time"
-
-	"github.com/kr/pretty"
-	"github.com/micro/go-micro/v3/store"
-)
-
-func TestSQL(t *testing.T) {
-	connection := fmt.Sprintf(
-		"host=%s port=%d user=%s sslmode=disable dbname=%s",
-		"localhost",
-		26257,
-		"root",
-		"test",
-	)
-	db, err := sql.Open("postgres", connection)
-	if err != nil {
-		t.Fatal(err)
-	}
-	if err := db.Ping(); err != nil {
-		t.Skip("store/cockroach: can't connect to db")
-	}
-	db.Close()
-
-	sqlStore := NewStore(
-		store.Database("testsql"),
-		store.Nodes(connection),
-	)
-
-	if err := sqlStore.Init(); err != nil {
-		t.Fatal(err)
-	}
-
-	keys, err := sqlStore.List()
-	if err != nil {
-		t.Error(err)
-	} else {
-		t.Logf("%# v\n", pretty.Formatter(keys))
-	}
-
-	err = sqlStore.Write(
-		&store.Record{
-			Key:   "test",
-			Value: []byte("foo"),
-		},
-	)
-	if err != nil {
-		t.Error(err)
-	}
-	err = sqlStore.Write(
-		&store.Record{
-			Key:   "bar",
-			Value: []byte("baz"),
-		},
-	)
-	if err != nil {
-		t.Error(err)
-	}
-	err = sqlStore.Write(
-		&store.Record{
-			Key:   "qux",
-			Value: []byte("aasad"),
-		},
-	)
-	if err != nil {
-		t.Error(err)
-	}
-	err = sqlStore.Delete("qux")
-	if err != nil {
-		t.Error(err)
-	}
-
-	err = sqlStore.Write(&store.Record{
-		Key:    "test",
-		Value:  []byte("bar"),
-		Expiry: time.Second * 10,
-	})
-	if err != nil {
-		t.Error(err)
-	}
-
-	records, err := sqlStore.Read("test")
-	if err != nil {
-		t.Error(err)
-	}
-	t.Logf("%# v\n", pretty.Formatter(records))
-	if string(records[0].Value) != "bar" {
-		t.Error("Expected bar, got ", string(records[0].Value))
-	}
-
-	time.Sleep(11 * time.Second)
-	_, err = sqlStore.Read("test")
-	switch err {
-	case nil:
-		t.Error("Key test should have expired")
-	default:
-		t.Error(err)
-	case store.ErrNotFound:
-		break
-	}
-	sqlStore.Delete("bar")
-	sqlStore.Write(&store.Record{Key: "aaa", Value: []byte("bbb"), Expiry: 10 * time.Second})
-	sqlStore.Write(&store.Record{Key: "aaaa", Value: []byte("bbb"), Expiry: 10 * time.Second})
-	sqlStore.Write(&store.Record{Key: "aaaaa", Value: []byte("bbb"), Expiry: 10 * time.Second})
-	results, err := sqlStore.Read("a", store.ReadPrefix())
-	if err != nil {
-		t.Error(err)
-	}
-	if len(results) != 3 {
-		t.Fatal("Results should have returned 3 records")
-	}
-	time.Sleep(10 * time.Second)
-	results, err = sqlStore.Read("a", store.ReadPrefix())
-	if err != nil {
-		t.Error(err)
-	}
-	if len(results) != 0 {
-		t.Fatal("Results should have returned 0 records")
-	}
-
-	sqlStore.Write(&store.Record{Key: "bbb", Value: []byte("bbb")}, store.WriteExpiry(time.Now().Add(10*time.Second)))
-	sqlStore.Write(&store.Record{Key: "bbbb", Value: []byte("bbb")}, store.WriteExpiry(time.Now().Add(10*time.Second)))
-	sqlStore.Write(&store.Record{Key: "bbbbb", Value: []byte("bbb")}, store.WriteExpiry(time.Now().Add(10*time.Second)))
-	results, err = sqlStore.Read("b", store.ReadPrefix())
-	if err != nil {
-		t.Error(err)
-	}
-	if len(results) != 3 {
-		t.Fatalf("Results should have returned 3 records. Received %d", len(results))
-	}
-	time.Sleep(10 * time.Second)
-	results, err = sqlStore.Read("b", store.ReadPrefix())
-	if err != nil {
-		t.Error(err)
-	}
-	if len(results) != 0 {
-		t.Fatalf("Results should have returned 0 records. Received %d", len(results))
-	}
-
-	sqlStore.Write(&store.Record{Key: "ccc", Value: []byte("bbb")}, store.WriteTTL(10*time.Second))
-	sqlStore.Write(&store.Record{Key: "cccc", Value: []byte("bbb")}, store.WriteTTL(10*time.Second))
-	sqlStore.Write(&store.Record{Key: "ccccc", Value: []byte("bbb")}, store.WriteTTL(10*time.Second))
-	results, err = sqlStore.Read("c", store.ReadPrefix())
-	if err != nil {
-		t.Error(err)
-	}
-	if len(results) != 3 {
-		t.Fatalf("Results should have returned 3 records. Received %d", len(results))
-	}
-	time.Sleep(10 * time.Second)
-	results, err = sqlStore.Read("c", store.ReadPrefix())
-	if err != nil {
-		t.Error(err)
-	}
-	if len(results) != 0 {
-		t.Fatalf("Results should have returned 0 records. Received %d", len(results))
-	}
-}

store/file/file.go

@@ -153,13 +153,18 @@ func (m *fileStore) list(db *bolt.DB, limit, offset uint) []string {
 
 	if limit != 0 || offset != 0 {
 		sort.Slice(allKeys, func(i, j int) bool { return allKeys[i] < allKeys[j] })
-		min := func(i, j uint) uint {
+		end := len(allKeys)
-			if i < j {
+		if limit > 0 {
-				return i
+			calcLimit := int(offset + limit)
+			if calcLimit < end {
+				end = calcLimit
 			}
-			return j
 		}
-		return allKeys[offset:min(limit, uint(len(allKeys)))]
+
+		if int(offset) >= end {
+			return nil
+		}
+		return allKeys[offset:end]
 	}
 
 	return allKeys

store/file/file_test.go

@@ -1,268 +0,0 @@
-package file
-
-import (
-	"fmt"
-	"os"
-	"path/filepath"
-	"strings"
-	"testing"
-	"time"
-
-	"github.com/davecgh/go-spew/spew"
-	"github.com/kr/pretty"
-	"github.com/micro/go-micro/v3/store"
-)
-
-func cleanup(db string, s store.Store) {
-	s.Close()
-	dir := filepath.Join(DefaultDir, db+"/")
-	os.RemoveAll(dir)
-}
-
-func TestFileStoreReInit(t *testing.T) {
-	s := NewStore(store.Table("aaa"))
-	defer cleanup(DefaultDatabase, s)
-	s.Init(store.Table("bbb"))
-	if s.Options().Table != "bbb" {
-		t.Error("Init didn't reinitialise the store")
-	}
-}
-
-func TestFileStoreBasic(t *testing.T) {
-	s := NewStore()
-	defer cleanup(DefaultDatabase, s)
-	fileTest(s, t)
-}
-
-func TestFileStoreTable(t *testing.T) {
-	s := NewStore(store.Table("testTable"))
-	defer cleanup(DefaultDatabase, s)
-	fileTest(s, t)
-}
-
-func TestFileStoreDatabase(t *testing.T) {
-	s := NewStore(store.Database("testdb"))
-	defer cleanup("testdb", s)
-	fileTest(s, t)
-}
-
-func TestFileStoreDatabaseTable(t *testing.T) {
-	s := NewStore(store.Table("testTable"), store.Database("testdb"))
-	defer cleanup("testdb", s)
-	fileTest(s, t)
-}
-
-func fileTest(s store.Store, t *testing.T) {
-	if len(os.Getenv("IN_TRAVIS_CI")) == 0 {
-		t.Logf("Options %s %v\n", s.String(), s.Options())
-	}
-	// Read and Write an expiring Record
-	if err := s.Write(&store.Record{
-		Key:    "Hello",
-		Value:  []byte("World"),
-		Expiry: time.Millisecond * 150,
-	}); err != nil {
-		t.Error(err)
-	}
-
-	if r, err := s.Read("Hello"); err != nil {
-		t.Fatal(err)
-	} else {
-		if len(r) != 1 {
-			t.Error("Read returned multiple records")
-		}
-		if r[0].Key != "Hello" {
-			t.Errorf("Expected %s, got %s", "Hello", r[0].Key)
-		}
-		if string(r[0].Value) != "World" {
-			t.Errorf("Expected %s, got %s", "World", r[0].Value)
-		}
-	}
-
-	// wait for expiry
-	time.Sleep(time.Millisecond * 200)
-
-	if _, err := s.Read("Hello"); err != store.ErrNotFound {
-		t.Errorf("Expected %# v, got %# v", store.ErrNotFound, err)
-	}
-
-	// Write 3 records with various expiry and get with Table
-	records := []*store.Record{
-		&store.Record{
-			Key:   "foo",
-			Value: []byte("foofoo"),
-		},
-		&store.Record{
-			Key:    "foobar",
-			Value:  []byte("foobarfoobar"),
-			Expiry: time.Millisecond * 100,
-		},
-	}
-
-	for _, r := range records {
-		if err := s.Write(r); err != nil {
-			t.Errorf("Couldn't write k: %s, v: %# v (%s)", r.Key, pretty.Formatter(r.Value), err)
-		}
-	}
-
-	if results, err := s.Read("foo", store.ReadPrefix()); err != nil {
-		t.Errorf("Couldn't read all \"foo\" keys, got %# v (%s)", spew.Sdump(results), err)
-	} else {
-		if len(results) != 2 {
-			t.Errorf("Expected 2 items, got %d", len(results))
-			//t.Logf("Table test: %v\n", spew.Sdump(results))
-		}
-	}
-
-	// wait for the expiry
-	time.Sleep(time.Millisecond * 200)
-
-	if results, err := s.Read("foo", store.ReadPrefix()); err != nil {
-		t.Errorf("Couldn't read all \"foo\" keys, got %# v (%s)", spew.Sdump(results), err)
-	} else if len(results) != 1 {
-		t.Errorf("Expected 1 item, got %d", len(results))
-		//t.Logf("Table test: %v\n", spew.Sdump(results))
-	}
-
-	if err := s.Delete("foo"); err != nil {
-		t.Errorf("Delete failed (%v)", err)
-	}
-
-	if results, err := s.Read("foo"); err != store.ErrNotFound {
-		t.Errorf("Expected read failure read all \"foo\" keys, got %# v (%s)", spew.Sdump(results), err)
-	} else {
-		if len(results) != 0 {
-			t.Errorf("Expected 0 items, got %d (%# v)", len(results), spew.Sdump(results))
-		}
-	}
-
-	// Write 3 records with various expiry and get with Suffix
-	records = []*store.Record{
-		&store.Record{
-			Key:   "foo",
-			Value: []byte("foofoo"),
-		},
-		&store.Record{
-			Key:   "barfoo",
-			Value: []byte("barfoobarfoo"),
-
-			Expiry: time.Millisecond * 100,
-		},
-		&store.Record{
-			Key:    "bazbarfoo",
-			Value:  []byte("bazbarfoobazbarfoo"),
-			Expiry: 2 * time.Millisecond * 100,
-		},
-	}
-	for _, r := range records {
-		if err := s.Write(r); err != nil {
-			t.Errorf("Couldn't write k: %s, v: %# v (%s)", r.Key, pretty.Formatter(r.Value), err)
-		}
-	}
-	if results, err := s.Read("foo", store.ReadSuffix()); err != nil {
-		t.Errorf("Couldn't read all \"foo\" keys, got %# v (%s)", spew.Sdump(results), err)
-	} else {
-		if len(results) != 3 {
-			t.Errorf("Expected 3 items, got %d", len(results))
-			//t.Logf("Table test: %v\n", spew.Sdump(results))
-		}
-
-	}
-	time.Sleep(time.Millisecond * 100)
-	if results, err := s.Read("foo", store.ReadSuffix()); err != nil {
-		t.Errorf("Couldn't read all \"foo\" keys, got %# v (%s)", spew.Sdump(results), err)
-	} else {
-		if len(results) != 2 {
-			t.Errorf("Expected 2 items, got %d", len(results))
-			//t.Logf("Table test: %v\n", spew.Sdump(results))
-		}
-
-	}
-	time.Sleep(time.Millisecond * 100)
-	if results, err := s.Read("foo", store.ReadSuffix()); err != nil {
-		t.Errorf("Couldn't read all \"foo\" keys, got %# v (%s)", spew.Sdump(results), err)
-	} else {
-		if len(results) != 1 {
-			t.Errorf("Expected 1 item, got %d", len(results))
-			//	t.Logf("Table test: %# v\n", spew.Sdump(results))
-		}
-	}
-	if err := s.Delete("foo"); err != nil {
-		t.Errorf("Delete failed (%v)", err)
-	}
-	if results, err := s.Read("foo", store.ReadSuffix()); err != nil {
-		t.Errorf("Couldn't read all \"foo\" keys, got %# v (%s)", spew.Sdump(results), err)
-	} else {
-		if len(results) != 0 {
-			t.Errorf("Expected 0 items, got %d (%# v)", len(results), spew.Sdump(results))
-		}
-	}
-
-	// Test Table, Suffix and WriteOptions
-	if err := s.Write(&store.Record{
-		Key:   "foofoobarbar",
-		Value: []byte("something"),
-	}, store.WriteTTL(time.Millisecond*100)); err != nil {
-		t.Error(err)
-	}
-	if err := s.Write(&store.Record{
-		Key:   "foofoo",
-		Value: []byte("something"),
-	}, store.WriteExpiry(time.Now().Add(time.Millisecond*100))); err != nil {
-		t.Error(err)
-	}
-	if err := s.Write(&store.Record{
-		Key:   "barbar",
-		Value: []byte("something"),
-		// TTL has higher precedence than expiry
-	}, store.WriteExpiry(time.Now().Add(time.Hour)), store.WriteTTL(time.Millisecond*100)); err != nil {
-		t.Error(err)
-	}
-
-	if results, err := s.Read("foo", store.ReadPrefix(), store.ReadSuffix()); err != nil {
-		t.Error(err)
-	} else {
-		if len(results) != 1 {
-			t.Errorf("Expected 1 results, got %d: %# v", len(results), spew.Sdump(results))
-		}
-	}
-
-	time.Sleep(time.Millisecond * 100)
-
-	if results, err := s.List(); err != nil {
-		t.Errorf("List failed: %s", err)
-	} else {
-		if len(results) != 0 {
-			t.Errorf("Expiry options were not effective, results :%v", spew.Sdump(results))
-		}
-	}
-
-	// write the following records
-	for i := 0; i < 10; i++ {
-		s.Write(&store.Record{
-			Key:   fmt.Sprintf("a%d", i),
-			Value: []byte{},
-		})
-	}
-
-	// read back a few records
-	if results, err := s.Read("a", store.ReadLimit(5), store.ReadPrefix()); err != nil {
-		t.Error(err)
-	} else {
-		if len(results) != 5 {
-			t.Fatal("Expected 5 results, got ", len(results))
-		}
-		if !strings.HasPrefix(results[0].Key, "a") {
-			t.Fatalf("Expected a prefix, got %s", results[0].Key)
-		}
-	}
-
-	// read the rest back
-	if results, err := s.Read("a", store.ReadLimit(30), store.ReadOffset(5), store.ReadPrefix()); err != nil {
-		t.Fatal(err)
-	} else {
-		if len(results) != 5 {
-			t.Fatal("Expected 5 results, got ", len(results))
-		}
-	}
-}

store/memory/memory.go

@@ -136,13 +136,19 @@ func (m *memoryStore) list(prefix string, limit, offset uint) []string {
 
 	if limit != 0 || offset != 0 {
 		sort.Slice(allKeys, func(i, j int) bool { return allKeys[i] < allKeys[j] })
-		min := func(i, j uint) uint {
+		sort.Slice(allKeys, func(i, j int) bool { return allKeys[i] < allKeys[j] })
-			if i < j {
+		end := len(allKeys)
-				return i
+		if limit > 0 {
+			calcLimit := int(offset + limit)
+			if calcLimit < end {
+				end = calcLimit
 			}
-			return j
 		}
-		return allKeys[offset:min(limit, uint(len(allKeys)))]
+
+		if int(offset) >= end {
+			return nil
+		}
+		return allKeys[offset:end]
 	}
 
 	return allKeys

store/memory/memory_test.go

@@ -1,278 +0,0 @@
-package memory
-
-import (
-	"fmt"
-	"os"
-	"testing"
-	"time"
-
-	"github.com/kr/pretty"
-	"github.com/micro/go-micro/v3/store"
-)
-
-func TestMemoryReInit(t *testing.T) {
-	s := NewStore(store.Table("aaa"))
-	s.Init(store.Table(""))
-	if len(s.Options().Table) > 0 {
-		t.Error("Init didn't reinitialise the store")
-	}
-}
-
-func TestMemoryBasic(t *testing.T) {
-	s := NewStore()
-	s.Init()
-	basictest(s, t)
-}
-
-func TestMemoryPrefix(t *testing.T) {
-	s := NewStore()
-	s.Init(store.Table("some-prefix"))
-	basictest(s, t)
-}
-
-func TestMemoryNamespace(t *testing.T) {
-	s := NewStore()
-	s.Init(store.Database("some-namespace"))
-	basictest(s, t)
-}
-
-func TestMemoryNamespacePrefix(t *testing.T) {
-	s := NewStore()
-	s.Init(store.Table("some-prefix"), store.Database("some-namespace"))
-	basictest(s, t)
-}
-
-func basictest(s store.Store, t *testing.T) {
-	if len(os.Getenv("IN_TRAVIS_CI")) == 0 {
-		t.Logf("Testing store %s, with options %# v\n", s.String(), pretty.Formatter(s.Options()))
-	}
-	// Read and Write an expiring Record
-	if err := s.Write(&store.Record{
-		Key:    "Hello",
-		Value:  []byte("World"),
-		Expiry: time.Millisecond * 100,
-	}); err != nil {
-		t.Error(err)
-	}
-	if r, err := s.Read("Hello"); err != nil {
-		t.Error(err)
-	} else {
-		if len(r) != 1 {
-			t.Error("Read returned multiple records")
-		}
-		if r[0].Key != "Hello" {
-			t.Errorf("Expected %s, got %s", "Hello", r[0].Key)
-		}
-		if string(r[0].Value) != "World" {
-			t.Errorf("Expected %s, got %s", "World", r[0].Value)
-		}
-	}
-	time.Sleep(time.Millisecond * 200)
-	if _, err := s.Read("Hello"); err != store.ErrNotFound {
-		t.Errorf("Expected %# v, got %# v", store.ErrNotFound, err)
-	}
-
-	// Write 3 records with various expiry and get with prefix
-	records := []*store.Record{
-		&store.Record{
-			Key:   "foo",
-			Value: []byte("foofoo"),
-		},
-		&store.Record{
-			Key:    "foobar",
-			Value:  []byte("foobarfoobar"),
-			Expiry: time.Millisecond * 100,
-		},
-		&store.Record{
-			Key:    "foobarbaz",
-			Value:  []byte("foobarbazfoobarbaz"),
-			Expiry: 2 * time.Millisecond * 100,
-		},
-	}
-	for _, r := range records {
-		if err := s.Write(r); err != nil {
-			t.Errorf("Couldn't write k: %s, v: %# v (%s)", r.Key, pretty.Formatter(r.Value), err)
-		}
-	}
-	if results, err := s.Read("foo", store.ReadPrefix()); err != nil {
-		t.Errorf("Couldn't read all \"foo\" keys, got %# v (%s)", pretty.Formatter(results), err)
-	} else {
-		if len(results) != 3 {
-			t.Errorf("Expected 3 items, got %d", len(results))
-		}
-		if len(os.Getenv("IN_TRAVIS_CI")) == 0 {
-			t.Logf("Prefix test: %v\n", pretty.Formatter(results))
-		}
-	}
-	time.Sleep(time.Millisecond * 100)
-	if results, err := s.Read("foo", store.ReadPrefix()); err != nil {
-		t.Errorf("Couldn't read all \"foo\" keys, got %# v (%s)", pretty.Formatter(results), err)
-	} else {
-		if len(results) != 2 {
-			t.Errorf("Expected 2 items, got %d", len(results))
-		}
-		if len(os.Getenv("IN_TRAVIS_CI")) == 0 {
-			t.Logf("Prefix test: %v\n", pretty.Formatter(results))
-		}
-	}
-	time.Sleep(time.Millisecond * 100)
-	if results, err := s.Read("foo", store.ReadPrefix()); err != nil {
-		t.Errorf("Couldn't read all \"foo\" keys, got %# v (%s)", pretty.Formatter(results), err)
-	} else {
-		if len(results) != 1 {
-			t.Errorf("Expected 1 item, got %d", len(results))
-		}
-		if len(os.Getenv("IN_TRAVIS_CI")) == 0 {
-			t.Logf("Prefix test: %# v\n", pretty.Formatter(results))
-		}
-	}
-	if err := s.Delete("foo", func(d *store.DeleteOptions) {}); err != nil {
-		t.Errorf("Delete failed (%v)", err)
-	}
-	if results, err := s.Read("foo", store.ReadPrefix()); err != nil {
-		t.Errorf("Couldn't read all \"foo\" keys, got %# v (%s)", pretty.Formatter(results), err)
-	} else {
-		if len(results) != 0 {
-			t.Errorf("Expected 0 items, got %d (%# v)", len(results), pretty.Formatter(results))
-		}
-	}
-
-	// Write 3 records with various expiry and get with Suffix
-	records = []*store.Record{
-		&store.Record{
-			Key:   "foo",
-			Value: []byte("foofoo"),
-		},
-		&store.Record{
-			Key:    "barfoo",
-			Value:  []byte("barfoobarfoo"),
-			Expiry: time.Millisecond * 100,
-		},
-		&store.Record{
-			Key:    "bazbarfoo",
-			Value:  []byte("bazbarfoobazbarfoo"),
-			Expiry: 2 * time.Millisecond * 100,
-		},
-	}
-	for _, r := range records {
-		if err := s.Write(r); err != nil {
-			t.Errorf("Couldn't write k: %s, v: %# v (%s)", r.Key, pretty.Formatter(r.Value), err)
-		}
-	}
-	if results, err := s.Read("foo", store.ReadSuffix()); err != nil {
-		t.Errorf("Couldn't read all \"foo\" keys, got %# v (%s)", pretty.Formatter(results), err)
-	} else {
-		if len(results) != 3 {
-			t.Errorf("Expected 3 items, got %d", len(results))
-		}
-		if len(os.Getenv("IN_TRAVIS_CI")) == 0 {
-			t.Logf("Prefix test: %v\n", pretty.Formatter(results))
-		}
-	}
-	time.Sleep(time.Millisecond * 100)
-	if results, err := s.Read("foo", store.ReadSuffix()); err != nil {
-		t.Errorf("Couldn't read all \"foo\" keys, got %# v (%s)", pretty.Formatter(results), err)
-	} else {
-		if len(results) != 2 {
-			t.Errorf("Expected 2 items, got %d", len(results))
-		}
-		if len(os.Getenv("IN_TRAVIS_CI")) == 0 {
-			t.Logf("Prefix test: %v\n", pretty.Formatter(results))
-		}
-	}
-	time.Sleep(time.Millisecond * 100)
-	if results, err := s.Read("foo", store.ReadSuffix()); err != nil {
-		t.Errorf("Couldn't read all \"foo\" keys, got %# v (%s)", pretty.Formatter(results), err)
-	} else {
-		if len(results) != 1 {
-			t.Errorf("Expected 1 item, got %d", len(results))
-		}
-		if len(os.Getenv("IN_TRAVIS_CI")) == 0 {
-			t.Logf("Prefix test: %# v\n", pretty.Formatter(results))
-		}
-	}
-	if err := s.Delete("foo"); err != nil {
-		t.Errorf("Delete failed (%v)", err)
-	}
-	if results, err := s.Read("foo", store.ReadSuffix()); err != nil {
-		t.Errorf("Couldn't read all \"foo\" keys, got %# v (%s)", pretty.Formatter(results), err)
-	} else {
-		if len(results) != 0 {
-			t.Errorf("Expected 0 items, got %d (%# v)", len(results), pretty.Formatter(results))
-		}
-	}
-
-	// Test Prefix, Suffix and WriteOptions
-	if err := s.Write(&store.Record{
-		Key:   "foofoobarbar",
-		Value: []byte("something"),
-	}, store.WriteTTL(time.Millisecond*100)); err != nil {
-		t.Error(err)
-	}
-	if err := s.Write(&store.Record{
-		Key:   "foofoo",
-		Value: []byte("something"),
-	}, store.WriteExpiry(time.Now().Add(time.Millisecond*100))); err != nil {
-		t.Error(err)
-	}
-	if err := s.Write(&store.Record{
-		Key:   "barbar",
-		Value: []byte("something"),
-		// TTL has higher precedence than expiry
-	}, store.WriteExpiry(time.Now().Add(time.Hour)), store.WriteTTL(time.Millisecond*100)); err != nil {
-		t.Error(err)
-	}
-	if results, err := s.Read("foo", store.ReadPrefix(), store.ReadSuffix()); err != nil {
-		t.Error(err)
-	} else {
-		if len(results) != 1 {
-			t.Errorf("Expected 1 results, got %d: %# v", len(results), pretty.Formatter(results))
-		}
-	}
-	time.Sleep(time.Millisecond * 100)
-	if results, err := s.List(); err != nil {
-		t.Errorf("List failed: %s", err)
-	} else {
-		if len(results) != 0 {
-			t.Error("Expiry options were not effective")
-		}
-	}
-	s.Write(&store.Record{Key: "a", Value: []byte("a")})
-	s.Write(&store.Record{Key: "aa", Value: []byte("aa")})
-	s.Write(&store.Record{Key: "aaa", Value: []byte("aaa")})
-	if results, err := s.Read("b", store.ReadPrefix()); err != nil {
-		t.Error(err)
-	} else {
-		if len(results) != 0 {
-			t.Errorf("Expected 0 results, got %d", len(results))
-		}
-	}
-
-	s.Close() // reset the store
-	for i := 0; i < 10; i++ {
-		s.Write(&store.Record{
-			Key:   fmt.Sprintf("a%d", i),
-			Value: []byte{},
-		})
-	}
-	if results, err := s.Read("a", store.ReadLimit(5), store.ReadPrefix()); err != nil {
-		t.Error(err)
-	} else {
-		if len(results) != 5 {
-			t.Fatal("Expected 5 results, got ", len(results))
-		}
-		if results[0].Key != "a0" {
-			t.Fatalf("Expected a0, got %s", results[0].Key)
-		}
-		if results[4].Key != "a4" {
-			t.Fatalf("Expected a4, got %s", results[4].Key)
-		}
-	}
-	if results, err := s.Read("a", store.ReadLimit(30), store.ReadOffset(5), store.ReadPrefix()); err != nil {
-		t.Error(err)
-	} else {
-		if len(results) != 5 {
-			t.Error("Expected 5 results, got ", len(results))
-		}
-	}
-}

store/test/store_test.go

@@ -0,0 +1,471 @@
+// Package test provides a way to run tests against all the various implementations of the Store interface.
+// It can't live in the store package itself because of circular import issues
+package test
+
+import (
+	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/micro/go-micro/v3/store/cache"
+
+	"github.com/micro/go-micro/v3/store/memory"
+
+	"github.com/kr/pretty"
+	"github.com/micro/go-micro/v3/store/cockroach"
+
+	"github.com/micro/go-micro/v3/store"
+
+	"github.com/micro/go-micro/v3/store/file"
+
+	"github.com/davecgh/go-spew/spew"
+)
+
+func fileStoreCleanup(db string, s store.Store) {
+	s.Close()
+	dir := filepath.Join(file.DefaultDir, db+"/")
+	os.RemoveAll(dir)
+}
+
+func cockroachStoreCleanup(db string, s store.Store) {
+	keys, _ := s.List()
+	for _, k := range keys {
+		s.Delete(k)
+	}
+	s.Close()
+}
+
+func memoryCleanup(db string, s store.Store) {
+	s.Close()
+}
+
+func cacheCleanup(db string, s store.Store) {
+	s.Close()
+}
+
+func TestStoreReInit(t *testing.T) {
+	tcs := []struct {
+		name    string
+		s       store.Store
+		cleanup func(db string, s store.Store)
+	}{
+		{name: "file", s: file.NewStore(store.Table("aaa")), cleanup: fileStoreCleanup},
+		{name: "cockroach", s: cockroach.NewStore(store.Table("aaa")), cleanup: cockroachStoreCleanup},
+		{name: "memory", s: memory.NewStore(store.Table("aaa")), cleanup: memoryCleanup},
+		{name: "cache", s: cache.NewStore(memory.NewStore(store.Table("aaa"))), cleanup: cacheCleanup},
+	}
+	for _, tc := range tcs {
+		t.Run(tc.name, func(t *testing.T) {
+			defer tc.cleanup(file.DefaultDatabase, tc.s)
+			tc.s.Init(store.Table("bbb"))
+			if tc.s.Options().Table != "bbb" {
+				t.Error("Init didn't reinitialise the store")
+			}
+		})
+	}
+}
+
+func TestStoreBasic(t *testing.T) {
+	tcs := []struct {
+		name    string
+		s       store.Store
+		cleanup func(db string, s store.Store)
+	}{
+		{name: "file", s: file.NewStore(), cleanup: fileStoreCleanup},
+		{name: "cockroach", s: cockroach.NewStore(), cleanup: cockroachStoreCleanup},
+		{name: "memory", s: memory.NewStore(), cleanup: memoryCleanup},
+		{name: "cache", s: cache.NewStore(memory.NewStore()), cleanup: cacheCleanup},
+	}
+	for _, tc := range tcs {
+		t.Run(tc.name, func(t *testing.T) {
+			defer tc.cleanup(file.DefaultDatabase, tc.s)
+			runStoreTest(tc.s, t)
+		})
+	}
+
+}
+
+func TestStoreTable(t *testing.T) {
+	tcs := []struct {
+		name    string
+		s       store.Store
+		cleanup func(db string, s store.Store)
+	}{
+		{name: "file", s: file.NewStore(store.Table("testTable")), cleanup: fileStoreCleanup},
+		{name: "cockroach", s: cockroach.NewStore(store.Table("testTable")), cleanup: cockroachStoreCleanup},
+		{name: "memory", s: memory.NewStore(store.Table("testTable")), cleanup: memoryCleanup},
+		{name: "cache", s: cache.NewStore(memory.NewStore(store.Table("testTable"))), cleanup: cacheCleanup},
+	}
+	for _, tc := range tcs {
+		t.Run(tc.name, func(t *testing.T) {
+			defer tc.cleanup(file.DefaultDatabase, tc.s)
+			runStoreTest(tc.s, t)
+		})
+	}
+}
+
+func TestStoreDatabase(t *testing.T) {
+	tcs := []struct {
+		name    string
+		s       store.Store
+		cleanup func(db string, s store.Store)
+	}{
+		{name: "file", s: file.NewStore(store.Database("testdb")), cleanup: fileStoreCleanup},
+		{name: "cockroach", s: cockroach.NewStore(store.Database("testdb")), cleanup: cockroachStoreCleanup},
+		{name: "memory", s: memory.NewStore(store.Database("testdb")), cleanup: memoryCleanup},
+		{name: "cache", s: cache.NewStore(memory.NewStore(store.Database("testdb"))), cleanup: cacheCleanup},
+	}
+	for _, tc := range tcs {
+		t.Run(tc.name, func(t *testing.T) {
+			defer tc.cleanup("testdb", tc.s)
+			runStoreTest(tc.s, t)
+		})
+	}
+}
+
+func TestStoreDatabaseTable(t *testing.T) {
+	tcs := []struct {
+		name    string
+		s       store.Store
+		cleanup func(db string, s store.Store)
+	}{
+		{name: "file", s: file.NewStore(store.Database("testdb"), store.Table("testTable")), cleanup: fileStoreCleanup},
+		{name: "cockroach", s: cockroach.NewStore(store.Database("testdb"), store.Table("testTable")), cleanup: cockroachStoreCleanup},
+		{name: "memory", s: memory.NewStore(store.Database("testdb"), store.Table("testTable")), cleanup: memoryCleanup},
+		{name: "cache", s: cache.NewStore(memory.NewStore(store.Database("testdb"), store.Table("testTable"))), cleanup: cacheCleanup},
+	}
+	for _, tc := range tcs {
+		t.Run(tc.name, func(t *testing.T) {
+			defer tc.cleanup("testdb", tc.s)
+			runStoreTest(tc.s, t)
+		})
+	}
+}
+
+func runStoreTest(s store.Store, t *testing.T) {
+	if len(os.Getenv("IN_TRAVIS_CI")) == 0 {
+		t.Logf("Options %s %v\n", s.String(), s.Options())
+	}
+
+	expiryTests(s, t)
+	suffixPrefixExpiryTests(s, t)
+	readTests(s, t)
+	listTests(s, t)
+
+}
+
+func readTests(s store.Store, t *testing.T) {
+	// Test Table, Suffix and WriteOptions
+	if err := s.Write(&store.Record{
+		Key:   "foofoobarbar",
+		Value: []byte("something"),
+	}, store.WriteTTL(time.Millisecond*100)); err != nil {
+		t.Error(err)
+	}
+	if err := s.Write(&store.Record{
+		Key:   "foofoo",
+		Value: []byte("something"),
+	}, store.WriteExpiry(time.Now().Add(time.Millisecond*100))); err != nil {
+		t.Error(err)
+	}
+	if err := s.Write(&store.Record{
+		Key:   "barbar",
+		Value: []byte("something"),
+		// TTL has higher precedence than expiry
+	}, store.WriteExpiry(time.Now().Add(time.Hour)), store.WriteTTL(time.Millisecond*100)); err != nil {
+		t.Error(err)
+	}
+
+	if results, err := s.Read("foo", store.ReadPrefix(), store.ReadSuffix()); err != nil {
+		t.Error(err)
+	} else {
+		if len(results) != 1 {
+			t.Errorf("Expected 1 results, got %d: %# v", len(results), spew.Sdump(results))
+		}
+	}
+
+	time.Sleep(time.Millisecond * 100)
+
+	if results, err := s.List(); err != nil {
+		t.Fatalf("List failed: %s", err)
+	} else {
+		if len(results) != 0 {
+			t.Fatalf("Expiry options were not effective, results :%v", spew.Sdump(results))
+		}
+	}
+
+	// write the following records
+	for i := 0; i < 10; i++ {
+		s.Write(&store.Record{
+			Key:   fmt.Sprintf("a%d", i),
+			Value: []byte{},
+		})
+	}
+
+	// read back a few records
+	if results, err := s.Read("a", store.ReadLimit(5), store.ReadPrefix()); err != nil {
+		t.Error(err)
+	} else {
+		if len(results) != 5 {
+			t.Fatal("Expected 5 results, got ", len(results))
+		}
+		if !strings.HasPrefix(results[0].Key, "a") {
+			t.Fatalf("Expected a prefix, got %s", results[0].Key)
+		}
+	}
+
+	// read the rest back
+	if results, err := s.Read("a", store.ReadLimit(30), store.ReadOffset(5), store.ReadPrefix()); err != nil {
+		t.Fatal(err)
+	} else {
+		if len(results) != 5 {
+			t.Fatal("Expected 5 results, got ", len(results))
+		}
+	}
+}
+
+func listTests(s store.Store, t *testing.T) {
+	for i := 0; i < 10; i++ {
+		s.Write(&store.Record{Key: fmt.Sprintf("List%d", i), Value: []byte("bar")})
+	}
+
+	recs, err := s.List(store.ListPrefix("List"))
+	if err != nil {
+		t.Fatalf("Error listing records %s", err)
+	}
+	if len(recs) != 10 {
+		t.Fatalf("Expected 10 records, received %d", len(recs))
+	}
+
+	recs, err = s.List(store.ListPrefix("List"), store.ListLimit(5))
+	if err != nil {
+		t.Fatalf("Error listing records %s", err)
+	}
+	if len(recs) != 5 {
+		t.Fatalf("Expected 5 records, received %d", len(recs))
+	}
+
+	recs, err = s.List(store.ListPrefix("List"), store.ListOffset(6))
+	if err != nil {
+		t.Fatalf("Error listing records %s", err)
+	}
+	if len(recs) != 4 {
+		t.Fatalf("Expected 4 records, received %d %+v", len(recs), recs)
+	}
+
+	recs, err = s.List(store.ListPrefix("List"), store.ListOffset(6), store.ListLimit(2))
+	if err != nil {
+		t.Fatalf("Error listing records %s", err)
+	}
+	if len(recs) != 2 {
+		t.Fatalf("Expected 2 records, received %d %+v", len(recs), recs)
+	}
+
+}
+
+func expiryTests(s store.Store, t *testing.T) {
+	// Read and Write an expiring Record
+	if err := s.Write(&store.Record{
+		Key:    "Hello",
+		Value:  []byte("World"),
+		Expiry: time.Millisecond * 150,
+	}); err != nil {
+		t.Error(err)
+	}
+
+	if r, err := s.Read("Hello"); err != nil {
+		t.Fatal(err)
+	} else {
+		if len(r) != 1 {
+			t.Error("Read returned multiple records")
+		}
+		if r[0].Key != "Hello" {
+			t.Errorf("Expected %s, got %s", "Hello", r[0].Key)
+		}
+		if string(r[0].Value) != "World" {
+			t.Errorf("Expected %s, got %s", "World", r[0].Value)
+		}
+	}
+
+	// wait for expiry
+	time.Sleep(time.Millisecond * 200)
+
+	if _, err := s.Read("Hello"); err != store.ErrNotFound {
+		t.Errorf("Expected %# v, got %# v", store.ErrNotFound, err)
+	}
+
+	// exercise the different ways to write an expiry, record.expiry
+	s.Write(&store.Record{Key: "aaa", Value: []byte("bbb"), Expiry: 1 * time.Second})
+	s.Write(&store.Record{Key: "aaaa", Value: []byte("bbb"), Expiry: 1 * time.Second})
+	s.Write(&store.Record{Key: "aaaaa", Value: []byte("bbb"), Expiry: 1 * time.Second})
+	results, err := s.Read("a", store.ReadPrefix())
+	if err != nil {
+		t.Error(err)
+	}
+	if len(results) != 3 {
+		t.Fatal("Results should have returned 3 records")
+	}
+	time.Sleep(1 * time.Second)
+	results, err = s.Read("a", store.ReadPrefix())
+	if err != nil {
+		t.Error(err)
+	}
+	if len(results) != 0 {
+		t.Fatal("Results should have returned 0 records")
+	}
+
+	// exercise the different ways to write an expiry, WriteExpiry
+	s.Write(&store.Record{Key: "bbb", Value: []byte("bbb")}, store.WriteExpiry(time.Now().Add(1*time.Second)))
+	s.Write(&store.Record{Key: "bbbb", Value: []byte("bbb")}, store.WriteExpiry(time.Now().Add(1*time.Second)))
+	s.Write(&store.Record{Key: "bbbbb", Value: []byte("bbb")}, store.WriteExpiry(time.Now().Add(1*time.Second)))
+	results, err = s.Read("b", store.ReadPrefix())
+	if err != nil {
+		t.Error(err)
+	}
+	if len(results) != 3 {
+		t.Fatalf("Results should have returned 3 records. Received %d", len(results))
+	}
+	time.Sleep(1 * time.Second)
+	results, err = s.Read("b", store.ReadPrefix())
+	if err != nil {
+		t.Error(err)
+	}
+	if len(results) != 0 {
+		t.Fatalf("Results should have returned 0 records. Received %d", len(results))
+	}
+
+	// exercise the different ways to write an expiry, WriteTTL
+	s.Write(&store.Record{Key: "ccc", Value: []byte("bbb")}, store.WriteTTL(1*time.Second))
+	s.Write(&store.Record{Key: "cccc", Value: []byte("bbb")}, store.WriteTTL(1*time.Second))
+	s.Write(&store.Record{Key: "ccccc", Value: []byte("bbb")}, store.WriteTTL(1*time.Second))
+	results, err = s.Read("c", store.ReadPrefix())
+	if err != nil {
+		t.Error(err)
+	}
+	if len(results) != 3 {
+		t.Fatalf("Results should have returned 3 records. Received %d", len(results))
+	}
+	time.Sleep(1 * time.Second)
+	results, err = s.Read("c", store.ReadPrefix())
+	if err != nil {
+		t.Error(err)
+	}
+	if len(results) != 0 {
+		t.Fatalf("Results should have returned 0 records. Received %d", len(results))
+	}
+}
+
+func suffixPrefixExpiryTests(s store.Store, t *testing.T) {
+	// Write 3 records with various expiry and get with Prefix
+	records := []*store.Record{
+		&store.Record{
+			Key:   "foo",
+			Value: []byte("foofoo"),
+		},
+		&store.Record{
+			Key:    "foobar",
+			Value:  []byte("foobarfoobar"),
+			Expiry: time.Millisecond * 100,
+		},
+	}
+
+	for _, r := range records {
+		if err := s.Write(r); err != nil {
+			t.Errorf("Couldn't write k: %s, v: %# v (%s)", r.Key, pretty.Formatter(r.Value), err)
+		}
+	}
+
+	if results, err := s.Read("foo", store.ReadPrefix()); err != nil {
+		t.Errorf("Couldn't read all \"foo\" keys, got %#v (%s)", spew.Sdump(results), err)
+	} else {
+		if len(results) != 2 {
+			t.Errorf("Expected 2 items, got %d", len(results))
+		}
+	}
+
+	// wait for the expiry
+	time.Sleep(time.Millisecond * 200)
+
+	if results, err := s.Read("foo", store.ReadPrefix()); err != nil {
+		t.Errorf("Couldn't read all \"foo\" keys, got %# v (%s)", spew.Sdump(results), err)
+	} else if len(results) != 1 {
+		t.Errorf("Expected 1 item, got %d", len(results))
+	}
+
+	if err := s.Delete("foo"); err != nil {
+		t.Errorf("Delete failed (%v)", err)
+	}
+
+	if results, err := s.Read("foo"); err != store.ErrNotFound {
+		t.Errorf("Expected read failure read all \"foo\" keys, got %# v (%s)", spew.Sdump(results), err)
+	} else {
+		if len(results) != 0 {
+			t.Errorf("Expected 0 items, got %d (%# v)", len(results), spew.Sdump(results))
+		}
+	}
+
+	// Write 3 records with various expiry and get with Suffix
+	records = []*store.Record{
+		&store.Record{
+			Key:   "foo",
+			Value: []byte("foofoo"),
+		},
+		&store.Record{
+			Key:   "barfoo",
+			Value: []byte("barfoobarfoo"),
+
+			Expiry: time.Millisecond * 100,
+		},
+		&store.Record{
+			Key:    "bazbarfoo",
+			Value:  []byte("bazbarfoobazbarfoo"),
+			Expiry: 2 * time.Millisecond * 100,
+		},
+	}
+	for _, r := range records {
+		if err := s.Write(r); err != nil {
+			t.Errorf("Couldn't write k: %s, v: %# v (%s)", r.Key, pretty.Formatter(r.Value), err)
+		}
+	}
+	if results, err := s.Read("foo", store.ReadSuffix()); err != nil {
+		t.Errorf("Couldn't read all \"foo\" keys, got %# v (%s)", spew.Sdump(results), err)
+	} else {
+		if len(results) != 3 {
+			t.Errorf("Expected 3 items, got %d", len(results))
+			//t.Logf("Table test: %v\n", spew.Sdump(results))
+		}
+
+	}
+	time.Sleep(time.Millisecond * 100)
+	if results, err := s.Read("foo", store.ReadSuffix()); err != nil {
+		t.Errorf("Couldn't read all \"foo\" keys, got %# v (%s)", spew.Sdump(results), err)
+	} else {
+		if len(results) != 2 {
+			t.Errorf("Expected 2 items, got %d", len(results))
+			//t.Logf("Table test: %v\n", spew.Sdump(results))
+		}
+
+	}
+	time.Sleep(time.Millisecond * 100)
+	if results, err := s.Read("foo", store.ReadSuffix()); err != nil {
+		t.Errorf("Couldn't read all \"foo\" keys, got %# v (%s)", spew.Sdump(results), err)
+	} else {
+		if len(results) != 1 {
+			t.Errorf("Expected 1 item, got %d", len(results))
+			//	t.Logf("Table test: %# v\n", spew.Sdump(results))
+		}
+	}
+	if err := s.Delete("foo"); err != nil {
+		t.Errorf("Delete failed (%v)", err)
+	}
+	if results, err := s.Read("foo", store.ReadSuffix()); err != nil {
+		t.Errorf("Couldn't read all \"foo\" keys, got %# v (%s)", spew.Sdump(results), err)
+	} else {
+		if len(results) != 0 {
+			t.Errorf("Expected 0 items, got %d (%# v)", len(results), spew.Sdump(results))
+		}
+	}
+}