From 47f356fc5fabe37fb498d9076f4cda420eb25f9a Mon Sep 17 00:00:00 2001 From: Dominic Wong Date: Wed, 19 Aug 2020 23:41:03 +0100 Subject: [PATCH] Unify the store tests (#1952) Add more tests for store --- .github/workflows/pr.yml | 5 +- go.sum | 16 - store/cockroach/cockroach.go | 148 +++++----- store/cockroach/cockroach_test.go | 163 ----------- store/file/file.go | 15 +- store/file/file_test.go | 268 ----------------- store/memory/memory.go | 16 +- store/memory/memory_test.go | 278 ------------------ store/test/store_test.go | 471 ++++++++++++++++++++++++++++++ 9 files changed, 575 insertions(+), 805 deletions(-) delete mode 100644 store/cockroach/cockroach_test.go delete mode 100644 store/file/file_test.go delete mode 100644 store/memory/memory_test.go create mode 100644 store/test/store_test.go diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index 59c4c27a..edf62196 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -25,4 +25,7 @@ jobs: id: tests env: IN_TRAVIS_CI: yes - run: go test -v ./... + run: | + wget -qO- https://binaries.cockroachdb.com/cockroach-v20.1.4.linux-amd64.tgz | tar xvz + cockroach-v20.1.4.linux-amd64/cockroach start-single-node --insecure & + go test -v ./... diff --git a/go.sum b/go.sum index 3ab7e668..00e60bd5 100644 --- a/go.sum +++ b/go.sum @@ -276,7 +276,6 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxv github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s= github.com/konsorten/go-windows-terminal-sequences v1.0.2/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= -github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0 h1:s5hAObm+yFO5uHYt5dYjxi2rXrsnmRpJx4OYvIWUaQs= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= @@ -306,7 +305,6 @@ github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5 github.com/miekg/dns v1.1.15/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.27 h1:aEH/kqUzUxGJ/UHcEKdJY+ugH6WEzsEBBSPa8zuy1aM= github.com/miekg/dns v1.1.27/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= -github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-vnc v0.0.0-20150629162542-723ed9867aed/go.mod h1:3rdaFaCv4AyBgu5ALFM0+tSuHrBh6v692nyQe3ikrq0= github.com/mitchellh/hashstructure v1.0.0 h1:ZkRJX1CyOoTkar7p/mLS5TZU4nJ1Rn/F8u9dGS02Q3Y= @@ -346,11 +344,9 @@ github.com/nrdcg/goinwx v0.6.1/go.mod h1:XPiut7enlbEdntAqalBIqcYcTEVhpv/dKWgDCX2 github.com/nrdcg/namesilo v0.2.1/go.mod h1:lwMvfQTyYq+BbjJd30ylEG4GPSS6PII0Tia4rRpRiyw= github.com/olekukonko/tablewriter v0.0.1/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= -github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.0 h1:Iw5WCbBcaAAd0fpRb1c9r5YCylv4XDoCSigm1zLevwU= github.com/onsi/ginkgo v1.12.0/go.mod h1:oUhWkIvk5aDxtKvDDuw8gItl8pKl42LzjC9KZE0HfGg= -github.com/onsi/gomega v1.4.3 h1:RE1xgDvH7imwFD45h+u2SgIfERHlS2yNG4DObb5BSKU= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.7.1 h1:K0jcRCwNQM3vFGh1ppMtDh/+7ApJrjldlX8fA0jDTLQ= github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY= @@ -385,21 +381,18 @@ github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXP github.com/prometheus/client_golang v0.9.2/go.mod h1:OsXs2jCmiKlQ1lTBmv21f2mNfw4xf/QclQDMrYNZzcM= github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= -github.com/prometheus/client_golang v1.1.0 h1:BQ53HtBmfOitExawJ6LokA4x8ov/z0SYYb0+HxJfRI8= github.com/prometheus/client_golang v1.1.0/go.mod h1:I1FGZT9+L76gKKOs5djB6ezCbFQP1xR9D75/vuwEF3g= github.com/prometheus/client_golang v1.7.0 h1:wCi7urQOGBsYcQROHqpUUX4ct84xp40t9R9JX0FuA/U= github.com/prometheus/client_golang v1.7.0/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4 h1:gQz4mCbXsO+nc9n1hCxHcGA3Zx3Eo+UHZoInFGUIXNM= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.0.0-20181126121408-4724e9255275/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= -github.com/prometheus/common v0.6.0 h1:kRhiuYSXR3+uv2IbVbZhUxK5zVD/2pp3Gd2PpvPkpEo= github.com/prometheus/common v0.6.0/go.mod h1:eBmuwkDJBwy6iBfxCBob6t6dR6ENT/y+J+Zk0j9GMYc= github.com/prometheus/common v0.10.0 h1:RyRA7RzGXQZiW+tGMr7sxa85G1z0yOpM1qq5c8lNawc= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= @@ -408,7 +401,6 @@ github.com/prometheus/procfs v0.0.0-20181204211112-1dc9a6cbc91a/go.mod h1:c3At6R github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.3/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= -github.com/prometheus/procfs v0.0.5 h1:3+auTFlqw+ZaQYJARz6ArODtkaIwtvBTx3N2NehQlL8= github.com/prometheus/procfs v0.0.5/go.mod h1:4A/X28fw3Fc593LaREMrKMqOKvUAntwMDaekg4FpcdQ= github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/prometheus/procfs v0.1.3 h1:F0+tqvhOksq22sc6iCHF5WGlWjdwj92p0udFh1VFBS8= @@ -417,11 +409,9 @@ github.com/rainycape/memcache v0.0.0-20150622160815-1031fa0ce2f2/go.mod h1:7tZKc github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/russross/blackfriday/v2 v2.0.1 h1:lPqVAte+HuHNfhJ/0LC98ESWRz8afy9tM/0RK8m9o+Q= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sacloud/libsacloud v1.26.1/go.mod h1:79ZwATmHLIFZIMd7sxA3LwzVy/B77uj3LDoToVTxDoQ= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= -github.com/shurcooL/sanitized_anchor_name v1.0.0 h1:PdmoCO6wvbs+7yrJyMORt4/BmY5IYyJwS/kOiWx8mHo= github.com/shurcooL/sanitized_anchor_name v1.0.0/go.mod h1:1NzhyTcUVG4SuEtjjoZeVRXNmyL/1OwPU0+IJeTBvfc= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= @@ -436,7 +426,6 @@ github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+ github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.4.0 h1:2E4SXV/wtOkTonXsotYi4li6zVWxYlZuYNCXe9XRJyk= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1 h1:nOGnQDM7FYENwehXlg/kFVnos3rEvtKTjRvOWSzb6H4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= @@ -610,7 +599,6 @@ golang.org/x/tools v0.0.0-20191029041327-9cc4af7d6b2c/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20191029190741-b9c20aec41a5/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191125144606-a911d9008d1f/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= golang.org/x/tools v0.0.0-20191216052735-49a3e744a425/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= -golang.org/x/tools v0.0.0-20191216173652-a0e659d51361 h1:RIIXAeV6GvDBuADKumTODatUqANFZ+5BPMnzsy4hulY= golang.org/x/tools v0.0.0-20191216173652-a0e659d51361/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200117065230-39095c1d176c h1:FodBYPZKH5tAN2O60HlglMwXGAeV/4k+NKbli79M/2c= golang.org/x/tools v0.0.0-20200117065230-39095c1d176c/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= @@ -639,7 +627,6 @@ google.golang.org/genproto v0.0.0-20190502173448-54afdca5d873/go.mod h1:VzzqZJRn google.golang.org/genproto v0.0.0-20190801165951-fa694d86fc64/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98Agz4BDEuKkezgsaosCRResVns1a3J2ZsMNc= google.golang.org/genproto v0.0.0-20190911173649-1774047e7e51/go.mod h1:IbNlFCBrqXvoKpeg0TB2l7cyZUmoaFKYIwrEpbDKLA8= -google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1 h1:aQktFqmDE2yjveXJlVIfslDFmFnUXSqG0i6KRcJAeMc= google.golang.org/genproto v0.0.0-20191216164720-4f79533eabd1/go.mod h1:n3cpQtvxv34hfy77yVDNjmbRyujviMdxYliBSkLhpCc= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 h1:+kGHl1aib/qcwaRi1CbqBZ1rk19r85MNUf8HaBghugY= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= @@ -650,7 +637,6 @@ google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= google.golang.org/protobuf v1.20.1-0.20200309200217-e05f789c0967/go.mod h1:A+miEFZTKqfCUM6K7xSMQL9OKL/b6hQv+e19PK+JZNE= google.golang.org/protobuf v1.21.0/go.mod h1:47Nbq4nVaFHyn7ilMalzfO3qCViNmqZ2kzikPIcrTAo= -google.golang.org/protobuf v1.22.0 h1:cJv5/xdbk1NnMPR1VP9+HU6gupuG9MLBoH1r6RHZ2MY= google.golang.org/protobuf v1.22.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= @@ -671,7 +657,6 @@ gopkg.in/ini.v1 v1.44.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ns1/ns1-go.v2 v2.0.0-20190730140822-b51389932cbc/go.mod h1:VV+3haRsgDiVLxyifmMBrBIuCWFBPYKbRssXB9z67Hw= gopkg.in/resty.v1 v1.9.1/go.mod h1:vo52Hzryw9PnPHcJfPsBiFW62XhNx5OczbV9y+IMpgc= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= -gopkg.in/square/go-jose.v2 v2.3.1 h1:SK5KegNXmKmqE342YYN2qPHEnUYeoMiXXl1poUlI+o4= gopkg.in/square/go-jose.v2 v2.3.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= gopkg.in/square/go-jose.v2 v2.4.1 h1:H0TmLt7/KmzlrDOpa1F+zr0Tk90PbJYBfsVUmRLrf9Y= gopkg.in/square/go-jose.v2 v2.4.1/go.mod h1:M9dMgbHiYLoDGQrXy7OpJDJWiKiU//h+vD76mk0e1AI= @@ -680,7 +665,6 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWD gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= -gopkg.in/yaml.v2 v2.2.4 h1:/eiJrUcujPVeJ3xlSWaiNi3uSVmDGBK1pDHUHAnao1I= gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.5/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8 h1:obN1ZagJSUGI0Ek/LBmuj4SNLPfIny3KsKFopxRdj10= diff --git a/store/cockroach/cockroach.go b/store/cockroach/cockroach.go index 2b2f3987..8a15c4da 100644 --- a/store/cockroach/cockroach.go +++ b/store/cockroach/cockroach.go @@ -27,7 +27,7 @@ var ( re = regexp.MustCompile("[^a-zA-Z0-9]+") statements = map[string]string{ - "list": "SELECT key, value, metadata, expiry FROM %s.%s;", + "list": "SELECT key, value, metadata, expiry FROM %s.%s WHERE key LIKE $1 ORDER BY key DESC LIMIT $2 OFFSET $3;", "read": "SELECT key, value, metadata, expiry FROM %s.%s WHERE key = $1;", "readMany": "SELECT key, value, metadata, expiry FROM %s.%s WHERE key LIKE $1;", "readOffset": "SELECT key, value, metadata, expiry FROM %s.%s WHERE key LIKE $1 ORDER BY key DESC LIMIT $2 OFFSET $3;", @@ -213,6 +213,23 @@ func (s *sqlStore) List(opts ...store.ListOption) ([]string, error) { if err := s.createDB(options.Database, options.Table); err != nil { return nil, err } + limit := sql.NullInt32{} + offset := 0 + pattern := "%" + if options.Prefix != "" || options.Suffix != "" { + if options.Prefix != "" { + pattern = options.Prefix + pattern + } + if options.Suffix != "" { + pattern = pattern + options.Suffix + } + } + if options.Offset > 0 { + offset = int(options.Offset) + } + if options.Limit > 0 { + limit = sql.NullInt32{Int32: int32(options.Limit), Valid: true} + } st, err := s.prepare(options.Database, options.Table, "list") if err != nil { @@ -220,7 +237,7 @@ func (s *sqlStore) List(opts ...store.ListOption) ([]string, error) { } defer st.Close() - rows, err := st.Query() + rows, err := st.Query(pattern, limit, offset) if err != nil { if err == sql.ErrNoRows { return nil, nil @@ -228,8 +245,55 @@ func (s *sqlStore) List(opts ...store.ListOption) ([]string, error) { return nil, err } defer rows.Close() - var keys []string + records, err := s.rowsToRecords(rows) + if err != nil { + return nil, err + } + for _, k := range records { + keys = append(keys, k.Key) + } + rowErr := rows.Close() + if rowErr != nil { + // transaction rollback or something + return keys, rowErr + } + if err := rows.Err(); err != nil { + return keys, err + } + return keys, nil +} + +// rowToRecord converts from sql.Row to a store.Record. If the record has expired it will issue a delete in a separate goroutine +func (s *sqlStore) rowToRecord(row *sql.Row) (*store.Record, error) { + var timehelper pq.NullTime + record := &store.Record{} + metadata := make(Metadata) + + if err := row.Scan(&record.Key, &record.Value, &metadata, &timehelper); err != nil { + if err == sql.ErrNoRows { + return record, store.ErrNotFound + } + return nil, err + } + + // set the metadata + record.Metadata = toMetadata(&metadata) + if timehelper.Valid { + if timehelper.Time.Before(time.Now()) { + // record has expired + go s.Delete(record.Key) + return nil, store.ErrNotFound + } + record.Expiry = time.Until(timehelper.Time) + + } + return record, nil +} + +// rowsToRecords converts from sql.Rows to []*store.Record. If a record has expired it will issue a delete in a separate goroutine +func (s *sqlStore) rowsToRecords(rows *sql.Rows) ([]*store.Record, error) { + var records []*store.Record var timehelper pq.NullTime for rows.Next() { @@ -237,7 +301,7 @@ func (s *sqlStore) List(opts ...store.ListOption) ([]string, error) { metadata := make(Metadata) if err := rows.Scan(&record.Key, &record.Value, &metadata, &timehelper); err != nil { - return keys, err + return records, err } // set the metadata @@ -249,22 +313,13 @@ func (s *sqlStore) List(opts ...store.ListOption) ([]string, error) { go s.Delete(record.Key) } else { record.Expiry = time.Until(timehelper.Time) - keys = append(keys, record.Key) + records = append(records, record) } } else { - keys = append(keys, record.Key) + records = append(records, record) } - } - rowErr := rows.Close() - if rowErr != nil { - // transaction rollback or something - return keys, rowErr - } - if err := rows.Err(); err != nil { - return keys, err - } - return keys, nil + return records, nil } // Read a single key @@ -283,9 +338,6 @@ func (s *sqlStore) Read(key string, opts ...store.ReadOption) ([]*store.Record, return s.read(key, options) } - var records []*store.Record - var timehelper pq.NullTime - st, err := s.prepare(options.Database, options.Table, "read") if err != nil { return nil, err @@ -293,32 +345,12 @@ func (s *sqlStore) Read(key string, opts ...store.ReadOption) ([]*store.Record, defer st.Close() row := st.QueryRow(key) - record := &store.Record{} - metadata := make(Metadata) - - if err := row.Scan(&record.Key, &record.Value, &metadata, &timehelper); err != nil { - if err == sql.ErrNoRows { - return records, store.ErrNotFound - } - return records, err + record, err := s.rowToRecord(row) + if err != nil { + return nil, err } - - // set the metadata - record.Metadata = toMetadata(&metadata) - - if timehelper.Valid { - if timehelper.Time.Before(time.Now()) { - // record has expired - go s.Delete(key) - return records, store.ErrNotFound - } - record.Expiry = time.Until(timehelper.Time) - records = append(records, record) - } else { - records = append(records, record) - } - - return records, nil + var records []*store.Record + return append(records, record), nil } // Read Many records @@ -361,31 +393,9 @@ func (s *sqlStore) read(key string, options store.ReadOptions) ([]*store.Record, defer rows.Close() - var records []*store.Record - var timehelper pq.NullTime - - for rows.Next() { - record := &store.Record{} - metadata := make(Metadata) - - if err := rows.Scan(&record.Key, &record.Value, &metadata, &timehelper); err != nil { - return records, err - } - - // set the metadata - record.Metadata = toMetadata(&metadata) - - if timehelper.Valid { - if timehelper.Time.Before(time.Now()) { - // record has expired - go s.Delete(record.Key) - } else { - record.Expiry = time.Until(timehelper.Time) - records = append(records, record) - } - } else { - records = append(records, record) - } + records, err := s.rowsToRecords(rows) + if err != nil { + return nil, err } rowErr := rows.Close() if rowErr != nil { diff --git a/store/cockroach/cockroach_test.go b/store/cockroach/cockroach_test.go deleted file mode 100644 index 2b794fb1..00000000 --- a/store/cockroach/cockroach_test.go +++ /dev/null @@ -1,163 +0,0 @@ -package cockroach - -import ( - "database/sql" - "fmt" - "testing" - "time" - - "github.com/kr/pretty" - "github.com/micro/go-micro/v3/store" -) - -func TestSQL(t *testing.T) { - connection := fmt.Sprintf( - "host=%s port=%d user=%s sslmode=disable dbname=%s", - "localhost", - 26257, - "root", - "test", - ) - db, err := sql.Open("postgres", connection) - if err != nil { - t.Fatal(err) - } - if err := db.Ping(); err != nil { - t.Skip("store/cockroach: can't connect to db") - } - db.Close() - - sqlStore := NewStore( - store.Database("testsql"), - store.Nodes(connection), - ) - - if err := sqlStore.Init(); err != nil { - t.Fatal(err) - } - - keys, err := sqlStore.List() - if err != nil { - t.Error(err) - } else { - t.Logf("%# v\n", pretty.Formatter(keys)) - } - - err = sqlStore.Write( - &store.Record{ - Key: "test", - Value: []byte("foo"), - }, - ) - if err != nil { - t.Error(err) - } - err = sqlStore.Write( - &store.Record{ - Key: "bar", - Value: []byte("baz"), - }, - ) - if err != nil { - t.Error(err) - } - err = sqlStore.Write( - &store.Record{ - Key: "qux", - Value: []byte("aasad"), - }, - ) - if err != nil { - t.Error(err) - } - err = sqlStore.Delete("qux") - if err != nil { - t.Error(err) - } - - err = sqlStore.Write(&store.Record{ - Key: "test", - Value: []byte("bar"), - Expiry: time.Second * 10, - }) - if err != nil { - t.Error(err) - } - - records, err := sqlStore.Read("test") - if err != nil { - t.Error(err) - } - t.Logf("%# v\n", pretty.Formatter(records)) - if string(records[0].Value) != "bar" { - t.Error("Expected bar, got ", string(records[0].Value)) - } - - time.Sleep(11 * time.Second) - _, err = sqlStore.Read("test") - switch err { - case nil: - t.Error("Key test should have expired") - default: - t.Error(err) - case store.ErrNotFound: - break - } - sqlStore.Delete("bar") - sqlStore.Write(&store.Record{Key: "aaa", Value: []byte("bbb"), Expiry: 10 * time.Second}) - sqlStore.Write(&store.Record{Key: "aaaa", Value: []byte("bbb"), Expiry: 10 * time.Second}) - sqlStore.Write(&store.Record{Key: "aaaaa", Value: []byte("bbb"), Expiry: 10 * time.Second}) - results, err := sqlStore.Read("a", store.ReadPrefix()) - if err != nil { - t.Error(err) - } - if len(results) != 3 { - t.Fatal("Results should have returned 3 records") - } - time.Sleep(10 * time.Second) - results, err = sqlStore.Read("a", store.ReadPrefix()) - if err != nil { - t.Error(err) - } - if len(results) != 0 { - t.Fatal("Results should have returned 0 records") - } - - sqlStore.Write(&store.Record{Key: "bbb", Value: []byte("bbb")}, store.WriteExpiry(time.Now().Add(10*time.Second))) - sqlStore.Write(&store.Record{Key: "bbbb", Value: []byte("bbb")}, store.WriteExpiry(time.Now().Add(10*time.Second))) - sqlStore.Write(&store.Record{Key: "bbbbb", Value: []byte("bbb")}, store.WriteExpiry(time.Now().Add(10*time.Second))) - results, err = sqlStore.Read("b", store.ReadPrefix()) - if err != nil { - t.Error(err) - } - if len(results) != 3 { - t.Fatalf("Results should have returned 3 records. Received %d", len(results)) - } - time.Sleep(10 * time.Second) - results, err = sqlStore.Read("b", store.ReadPrefix()) - if err != nil { - t.Error(err) - } - if len(results) != 0 { - t.Fatalf("Results should have returned 0 records. Received %d", len(results)) - } - - sqlStore.Write(&store.Record{Key: "ccc", Value: []byte("bbb")}, store.WriteTTL(10*time.Second)) - sqlStore.Write(&store.Record{Key: "cccc", Value: []byte("bbb")}, store.WriteTTL(10*time.Second)) - sqlStore.Write(&store.Record{Key: "ccccc", Value: []byte("bbb")}, store.WriteTTL(10*time.Second)) - results, err = sqlStore.Read("c", store.ReadPrefix()) - if err != nil { - t.Error(err) - } - if len(results) != 3 { - t.Fatalf("Results should have returned 3 records. Received %d", len(results)) - } - time.Sleep(10 * time.Second) - results, err = sqlStore.Read("c", store.ReadPrefix()) - if err != nil { - t.Error(err) - } - if len(results) != 0 { - t.Fatalf("Results should have returned 0 records. Received %d", len(results)) - } -} diff --git a/store/file/file.go b/store/file/file.go index 11cbfdd1..7e2b63e6 100644 --- a/store/file/file.go +++ b/store/file/file.go @@ -153,13 +153,18 @@ func (m *fileStore) list(db *bolt.DB, limit, offset uint) []string { if limit != 0 || offset != 0 { sort.Slice(allKeys, func(i, j int) bool { return allKeys[i] < allKeys[j] }) - min := func(i, j uint) uint { - if i < j { - return i + end := len(allKeys) + if limit > 0 { + calcLimit := int(offset + limit) + if calcLimit < end { + end = calcLimit } - return j } - return allKeys[offset:min(limit, uint(len(allKeys)))] + + if int(offset) >= end { + return nil + } + return allKeys[offset:end] } return allKeys diff --git a/store/file/file_test.go b/store/file/file_test.go deleted file mode 100644 index 7f9cf7c8..00000000 --- a/store/file/file_test.go +++ /dev/null @@ -1,268 +0,0 @@ -package file - -import ( - "fmt" - "os" - "path/filepath" - "strings" - "testing" - "time" - - "github.com/davecgh/go-spew/spew" - "github.com/kr/pretty" - "github.com/micro/go-micro/v3/store" -) - -func cleanup(db string, s store.Store) { - s.Close() - dir := filepath.Join(DefaultDir, db+"/") - os.RemoveAll(dir) -} - -func TestFileStoreReInit(t *testing.T) { - s := NewStore(store.Table("aaa")) - defer cleanup(DefaultDatabase, s) - s.Init(store.Table("bbb")) - if s.Options().Table != "bbb" { - t.Error("Init didn't reinitialise the store") - } -} - -func TestFileStoreBasic(t *testing.T) { - s := NewStore() - defer cleanup(DefaultDatabase, s) - fileTest(s, t) -} - -func TestFileStoreTable(t *testing.T) { - s := NewStore(store.Table("testTable")) - defer cleanup(DefaultDatabase, s) - fileTest(s, t) -} - -func TestFileStoreDatabase(t *testing.T) { - s := NewStore(store.Database("testdb")) - defer cleanup("testdb", s) - fileTest(s, t) -} - -func TestFileStoreDatabaseTable(t *testing.T) { - s := NewStore(store.Table("testTable"), store.Database("testdb")) - defer cleanup("testdb", s) - fileTest(s, t) -} - -func fileTest(s store.Store, t *testing.T) { - if len(os.Getenv("IN_TRAVIS_CI")) == 0 { - t.Logf("Options %s %v\n", s.String(), s.Options()) - } - // Read and Write an expiring Record - if err := s.Write(&store.Record{ - Key: "Hello", - Value: []byte("World"), - Expiry: time.Millisecond * 150, - }); err != nil { - t.Error(err) - } - - if r, err := s.Read("Hello"); err != nil { - t.Fatal(err) - } else { - if len(r) != 1 { - t.Error("Read returned multiple records") - } - if r[0].Key != "Hello" { - t.Errorf("Expected %s, got %s", "Hello", r[0].Key) - } - if string(r[0].Value) != "World" { - t.Errorf("Expected %s, got %s", "World", r[0].Value) - } - } - - // wait for expiry - time.Sleep(time.Millisecond * 200) - - if _, err := s.Read("Hello"); err != store.ErrNotFound { - t.Errorf("Expected %# v, got %# v", store.ErrNotFound, err) - } - - // Write 3 records with various expiry and get with Table - records := []*store.Record{ - &store.Record{ - Key: "foo", - Value: []byte("foofoo"), - }, - &store.Record{ - Key: "foobar", - Value: []byte("foobarfoobar"), - Expiry: time.Millisecond * 100, - }, - } - - for _, r := range records { - if err := s.Write(r); err != nil { - t.Errorf("Couldn't write k: %s, v: %# v (%s)", r.Key, pretty.Formatter(r.Value), err) - } - } - - if results, err := s.Read("foo", store.ReadPrefix()); err != nil { - t.Errorf("Couldn't read all \"foo\" keys, got %# v (%s)", spew.Sdump(results), err) - } else { - if len(results) != 2 { - t.Errorf("Expected 2 items, got %d", len(results)) - //t.Logf("Table test: %v\n", spew.Sdump(results)) - } - } - - // wait for the expiry - time.Sleep(time.Millisecond * 200) - - if results, err := s.Read("foo", store.ReadPrefix()); err != nil { - t.Errorf("Couldn't read all \"foo\" keys, got %# v (%s)", spew.Sdump(results), err) - } else if len(results) != 1 { - t.Errorf("Expected 1 item, got %d", len(results)) - //t.Logf("Table test: %v\n", spew.Sdump(results)) - } - - if err := s.Delete("foo"); err != nil { - t.Errorf("Delete failed (%v)", err) - } - - if results, err := s.Read("foo"); err != store.ErrNotFound { - t.Errorf("Expected read failure read all \"foo\" keys, got %# v (%s)", spew.Sdump(results), err) - } else { - if len(results) != 0 { - t.Errorf("Expected 0 items, got %d (%# v)", len(results), spew.Sdump(results)) - } - } - - // Write 3 records with various expiry and get with Suffix - records = []*store.Record{ - &store.Record{ - Key: "foo", - Value: []byte("foofoo"), - }, - &store.Record{ - Key: "barfoo", - Value: []byte("barfoobarfoo"), - - Expiry: time.Millisecond * 100, - }, - &store.Record{ - Key: "bazbarfoo", - Value: []byte("bazbarfoobazbarfoo"), - Expiry: 2 * time.Millisecond * 100, - }, - } - for _, r := range records { - if err := s.Write(r); err != nil { - t.Errorf("Couldn't write k: %s, v: %# v (%s)", r.Key, pretty.Formatter(r.Value), err) - } - } - if results, err := s.Read("foo", store.ReadSuffix()); err != nil { - t.Errorf("Couldn't read all \"foo\" keys, got %# v (%s)", spew.Sdump(results), err) - } else { - if len(results) != 3 { - t.Errorf("Expected 3 items, got %d", len(results)) - //t.Logf("Table test: %v\n", spew.Sdump(results)) - } - - } - time.Sleep(time.Millisecond * 100) - if results, err := s.Read("foo", store.ReadSuffix()); err != nil { - t.Errorf("Couldn't read all \"foo\" keys, got %# v (%s)", spew.Sdump(results), err) - } else { - if len(results) != 2 { - t.Errorf("Expected 2 items, got %d", len(results)) - //t.Logf("Table test: %v\n", spew.Sdump(results)) - } - - } - time.Sleep(time.Millisecond * 100) - if results, err := s.Read("foo", store.ReadSuffix()); err != nil { - t.Errorf("Couldn't read all \"foo\" keys, got %# v (%s)", spew.Sdump(results), err) - } else { - if len(results) != 1 { - t.Errorf("Expected 1 item, got %d", len(results)) - // t.Logf("Table test: %# v\n", spew.Sdump(results)) - } - } - if err := s.Delete("foo"); err != nil { - t.Errorf("Delete failed (%v)", err) - } - if results, err := s.Read("foo", store.ReadSuffix()); err != nil { - t.Errorf("Couldn't read all \"foo\" keys, got %# v (%s)", spew.Sdump(results), err) - } else { - if len(results) != 0 { - t.Errorf("Expected 0 items, got %d (%# v)", len(results), spew.Sdump(results)) - } - } - - // Test Table, Suffix and WriteOptions - if err := s.Write(&store.Record{ - Key: "foofoobarbar", - Value: []byte("something"), - }, store.WriteTTL(time.Millisecond*100)); err != nil { - t.Error(err) - } - if err := s.Write(&store.Record{ - Key: "foofoo", - Value: []byte("something"), - }, store.WriteExpiry(time.Now().Add(time.Millisecond*100))); err != nil { - t.Error(err) - } - if err := s.Write(&store.Record{ - Key: "barbar", - Value: []byte("something"), - // TTL has higher precedence than expiry - }, store.WriteExpiry(time.Now().Add(time.Hour)), store.WriteTTL(time.Millisecond*100)); err != nil { - t.Error(err) - } - - if results, err := s.Read("foo", store.ReadPrefix(), store.ReadSuffix()); err != nil { - t.Error(err) - } else { - if len(results) != 1 { - t.Errorf("Expected 1 results, got %d: %# v", len(results), spew.Sdump(results)) - } - } - - time.Sleep(time.Millisecond * 100) - - if results, err := s.List(); err != nil { - t.Errorf("List failed: %s", err) - } else { - if len(results) != 0 { - t.Errorf("Expiry options were not effective, results :%v", spew.Sdump(results)) - } - } - - // write the following records - for i := 0; i < 10; i++ { - s.Write(&store.Record{ - Key: fmt.Sprintf("a%d", i), - Value: []byte{}, - }) - } - - // read back a few records - if results, err := s.Read("a", store.ReadLimit(5), store.ReadPrefix()); err != nil { - t.Error(err) - } else { - if len(results) != 5 { - t.Fatal("Expected 5 results, got ", len(results)) - } - if !strings.HasPrefix(results[0].Key, "a") { - t.Fatalf("Expected a prefix, got %s", results[0].Key) - } - } - - // read the rest back - if results, err := s.Read("a", store.ReadLimit(30), store.ReadOffset(5), store.ReadPrefix()); err != nil { - t.Fatal(err) - } else { - if len(results) != 5 { - t.Fatal("Expected 5 results, got ", len(results)) - } - } -} diff --git a/store/memory/memory.go b/store/memory/memory.go index 20c190ca..cca91815 100644 --- a/store/memory/memory.go +++ b/store/memory/memory.go @@ -136,13 +136,19 @@ func (m *memoryStore) list(prefix string, limit, offset uint) []string { if limit != 0 || offset != 0 { sort.Slice(allKeys, func(i, j int) bool { return allKeys[i] < allKeys[j] }) - min := func(i, j uint) uint { - if i < j { - return i + sort.Slice(allKeys, func(i, j int) bool { return allKeys[i] < allKeys[j] }) + end := len(allKeys) + if limit > 0 { + calcLimit := int(offset + limit) + if calcLimit < end { + end = calcLimit } - return j } - return allKeys[offset:min(limit, uint(len(allKeys)))] + + if int(offset) >= end { + return nil + } + return allKeys[offset:end] } return allKeys diff --git a/store/memory/memory_test.go b/store/memory/memory_test.go deleted file mode 100644 index 59dba5d5..00000000 --- a/store/memory/memory_test.go +++ /dev/null @@ -1,278 +0,0 @@ -package memory - -import ( - "fmt" - "os" - "testing" - "time" - - "github.com/kr/pretty" - "github.com/micro/go-micro/v3/store" -) - -func TestMemoryReInit(t *testing.T) { - s := NewStore(store.Table("aaa")) - s.Init(store.Table("")) - if len(s.Options().Table) > 0 { - t.Error("Init didn't reinitialise the store") - } -} - -func TestMemoryBasic(t *testing.T) { - s := NewStore() - s.Init() - basictest(s, t) -} - -func TestMemoryPrefix(t *testing.T) { - s := NewStore() - s.Init(store.Table("some-prefix")) - basictest(s, t) -} - -func TestMemoryNamespace(t *testing.T) { - s := NewStore() - s.Init(store.Database("some-namespace")) - basictest(s, t) -} - -func TestMemoryNamespacePrefix(t *testing.T) { - s := NewStore() - s.Init(store.Table("some-prefix"), store.Database("some-namespace")) - basictest(s, t) -} - -func basictest(s store.Store, t *testing.T) { - if len(os.Getenv("IN_TRAVIS_CI")) == 0 { - t.Logf("Testing store %s, with options %# v\n", s.String(), pretty.Formatter(s.Options())) - } - // Read and Write an expiring Record - if err := s.Write(&store.Record{ - Key: "Hello", - Value: []byte("World"), - Expiry: time.Millisecond * 100, - }); err != nil { - t.Error(err) - } - if r, err := s.Read("Hello"); err != nil { - t.Error(err) - } else { - if len(r) != 1 { - t.Error("Read returned multiple records") - } - if r[0].Key != "Hello" { - t.Errorf("Expected %s, got %s", "Hello", r[0].Key) - } - if string(r[0].Value) != "World" { - t.Errorf("Expected %s, got %s", "World", r[0].Value) - } - } - time.Sleep(time.Millisecond * 200) - if _, err := s.Read("Hello"); err != store.ErrNotFound { - t.Errorf("Expected %# v, got %# v", store.ErrNotFound, err) - } - - // Write 3 records with various expiry and get with prefix - records := []*store.Record{ - &store.Record{ - Key: "foo", - Value: []byte("foofoo"), - }, - &store.Record{ - Key: "foobar", - Value: []byte("foobarfoobar"), - Expiry: time.Millisecond * 100, - }, - &store.Record{ - Key: "foobarbaz", - Value: []byte("foobarbazfoobarbaz"), - Expiry: 2 * time.Millisecond * 100, - }, - } - for _, r := range records { - if err := s.Write(r); err != nil { - t.Errorf("Couldn't write k: %s, v: %# v (%s)", r.Key, pretty.Formatter(r.Value), err) - } - } - if results, err := s.Read("foo", store.ReadPrefix()); err != nil { - t.Errorf("Couldn't read all \"foo\" keys, got %# v (%s)", pretty.Formatter(results), err) - } else { - if len(results) != 3 { - t.Errorf("Expected 3 items, got %d", len(results)) - } - if len(os.Getenv("IN_TRAVIS_CI")) == 0 { - t.Logf("Prefix test: %v\n", pretty.Formatter(results)) - } - } - time.Sleep(time.Millisecond * 100) - if results, err := s.Read("foo", store.ReadPrefix()); err != nil { - t.Errorf("Couldn't read all \"foo\" keys, got %# v (%s)", pretty.Formatter(results), err) - } else { - if len(results) != 2 { - t.Errorf("Expected 2 items, got %d", len(results)) - } - if len(os.Getenv("IN_TRAVIS_CI")) == 0 { - t.Logf("Prefix test: %v\n", pretty.Formatter(results)) - } - } - time.Sleep(time.Millisecond * 100) - if results, err := s.Read("foo", store.ReadPrefix()); err != nil { - t.Errorf("Couldn't read all \"foo\" keys, got %# v (%s)", pretty.Formatter(results), err) - } else { - if len(results) != 1 { - t.Errorf("Expected 1 item, got %d", len(results)) - } - if len(os.Getenv("IN_TRAVIS_CI")) == 0 { - t.Logf("Prefix test: %# v\n", pretty.Formatter(results)) - } - } - if err := s.Delete("foo", func(d *store.DeleteOptions) {}); err != nil { - t.Errorf("Delete failed (%v)", err) - } - if results, err := s.Read("foo", store.ReadPrefix()); err != nil { - t.Errorf("Couldn't read all \"foo\" keys, got %# v (%s)", pretty.Formatter(results), err) - } else { - if len(results) != 0 { - t.Errorf("Expected 0 items, got %d (%# v)", len(results), pretty.Formatter(results)) - } - } - - // Write 3 records with various expiry and get with Suffix - records = []*store.Record{ - &store.Record{ - Key: "foo", - Value: []byte("foofoo"), - }, - &store.Record{ - Key: "barfoo", - Value: []byte("barfoobarfoo"), - Expiry: time.Millisecond * 100, - }, - &store.Record{ - Key: "bazbarfoo", - Value: []byte("bazbarfoobazbarfoo"), - Expiry: 2 * time.Millisecond * 100, - }, - } - for _, r := range records { - if err := s.Write(r); err != nil { - t.Errorf("Couldn't write k: %s, v: %# v (%s)", r.Key, pretty.Formatter(r.Value), err) - } - } - if results, err := s.Read("foo", store.ReadSuffix()); err != nil { - t.Errorf("Couldn't read all \"foo\" keys, got %# v (%s)", pretty.Formatter(results), err) - } else { - if len(results) != 3 { - t.Errorf("Expected 3 items, got %d", len(results)) - } - if len(os.Getenv("IN_TRAVIS_CI")) == 0 { - t.Logf("Prefix test: %v\n", pretty.Formatter(results)) - } - } - time.Sleep(time.Millisecond * 100) - if results, err := s.Read("foo", store.ReadSuffix()); err != nil { - t.Errorf("Couldn't read all \"foo\" keys, got %# v (%s)", pretty.Formatter(results), err) - } else { - if len(results) != 2 { - t.Errorf("Expected 2 items, got %d", len(results)) - } - if len(os.Getenv("IN_TRAVIS_CI")) == 0 { - t.Logf("Prefix test: %v\n", pretty.Formatter(results)) - } - } - time.Sleep(time.Millisecond * 100) - if results, err := s.Read("foo", store.ReadSuffix()); err != nil { - t.Errorf("Couldn't read all \"foo\" keys, got %# v (%s)", pretty.Formatter(results), err) - } else { - if len(results) != 1 { - t.Errorf("Expected 1 item, got %d", len(results)) - } - if len(os.Getenv("IN_TRAVIS_CI")) == 0 { - t.Logf("Prefix test: %# v\n", pretty.Formatter(results)) - } - } - if err := s.Delete("foo"); err != nil { - t.Errorf("Delete failed (%v)", err) - } - if results, err := s.Read("foo", store.ReadSuffix()); err != nil { - t.Errorf("Couldn't read all \"foo\" keys, got %# v (%s)", pretty.Formatter(results), err) - } else { - if len(results) != 0 { - t.Errorf("Expected 0 items, got %d (%# v)", len(results), pretty.Formatter(results)) - } - } - - // Test Prefix, Suffix and WriteOptions - if err := s.Write(&store.Record{ - Key: "foofoobarbar", - Value: []byte("something"), - }, store.WriteTTL(time.Millisecond*100)); err != nil { - t.Error(err) - } - if err := s.Write(&store.Record{ - Key: "foofoo", - Value: []byte("something"), - }, store.WriteExpiry(time.Now().Add(time.Millisecond*100))); err != nil { - t.Error(err) - } - if err := s.Write(&store.Record{ - Key: "barbar", - Value: []byte("something"), - // TTL has higher precedence than expiry - }, store.WriteExpiry(time.Now().Add(time.Hour)), store.WriteTTL(time.Millisecond*100)); err != nil { - t.Error(err) - } - if results, err := s.Read("foo", store.ReadPrefix(), store.ReadSuffix()); err != nil { - t.Error(err) - } else { - if len(results) != 1 { - t.Errorf("Expected 1 results, got %d: %# v", len(results), pretty.Formatter(results)) - } - } - time.Sleep(time.Millisecond * 100) - if results, err := s.List(); err != nil { - t.Errorf("List failed: %s", err) - } else { - if len(results) != 0 { - t.Error("Expiry options were not effective") - } - } - s.Write(&store.Record{Key: "a", Value: []byte("a")}) - s.Write(&store.Record{Key: "aa", Value: []byte("aa")}) - s.Write(&store.Record{Key: "aaa", Value: []byte("aaa")}) - if results, err := s.Read("b", store.ReadPrefix()); err != nil { - t.Error(err) - } else { - if len(results) != 0 { - t.Errorf("Expected 0 results, got %d", len(results)) - } - } - - s.Close() // reset the store - for i := 0; i < 10; i++ { - s.Write(&store.Record{ - Key: fmt.Sprintf("a%d", i), - Value: []byte{}, - }) - } - if results, err := s.Read("a", store.ReadLimit(5), store.ReadPrefix()); err != nil { - t.Error(err) - } else { - if len(results) != 5 { - t.Fatal("Expected 5 results, got ", len(results)) - } - if results[0].Key != "a0" { - t.Fatalf("Expected a0, got %s", results[0].Key) - } - if results[4].Key != "a4" { - t.Fatalf("Expected a4, got %s", results[4].Key) - } - } - if results, err := s.Read("a", store.ReadLimit(30), store.ReadOffset(5), store.ReadPrefix()); err != nil { - t.Error(err) - } else { - if len(results) != 5 { - t.Error("Expected 5 results, got ", len(results)) - } - } -} diff --git a/store/test/store_test.go b/store/test/store_test.go new file mode 100644 index 00000000..b440c978 --- /dev/null +++ b/store/test/store_test.go @@ -0,0 +1,471 @@ +// Package test provides a way to run tests against all the various implementations of the Store interface. +// It can't live in the store package itself because of circular import issues +package test + +import ( + "fmt" + "os" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/micro/go-micro/v3/store/cache" + + "github.com/micro/go-micro/v3/store/memory" + + "github.com/kr/pretty" + "github.com/micro/go-micro/v3/store/cockroach" + + "github.com/micro/go-micro/v3/store" + + "github.com/micro/go-micro/v3/store/file" + + "github.com/davecgh/go-spew/spew" +) + +func fileStoreCleanup(db string, s store.Store) { + s.Close() + dir := filepath.Join(file.DefaultDir, db+"/") + os.RemoveAll(dir) +} + +func cockroachStoreCleanup(db string, s store.Store) { + keys, _ := s.List() + for _, k := range keys { + s.Delete(k) + } + s.Close() +} + +func memoryCleanup(db string, s store.Store) { + s.Close() +} + +func cacheCleanup(db string, s store.Store) { + s.Close() +} + +func TestStoreReInit(t *testing.T) { + tcs := []struct { + name string + s store.Store + cleanup func(db string, s store.Store) + }{ + {name: "file", s: file.NewStore(store.Table("aaa")), cleanup: fileStoreCleanup}, + {name: "cockroach", s: cockroach.NewStore(store.Table("aaa")), cleanup: cockroachStoreCleanup}, + {name: "memory", s: memory.NewStore(store.Table("aaa")), cleanup: memoryCleanup}, + {name: "cache", s: cache.NewStore(memory.NewStore(store.Table("aaa"))), cleanup: cacheCleanup}, + } + for _, tc := range tcs { + t.Run(tc.name, func(t *testing.T) { + defer tc.cleanup(file.DefaultDatabase, tc.s) + tc.s.Init(store.Table("bbb")) + if tc.s.Options().Table != "bbb" { + t.Error("Init didn't reinitialise the store") + } + }) + } +} + +func TestStoreBasic(t *testing.T) { + tcs := []struct { + name string + s store.Store + cleanup func(db string, s store.Store) + }{ + {name: "file", s: file.NewStore(), cleanup: fileStoreCleanup}, + {name: "cockroach", s: cockroach.NewStore(), cleanup: cockroachStoreCleanup}, + {name: "memory", s: memory.NewStore(), cleanup: memoryCleanup}, + {name: "cache", s: cache.NewStore(memory.NewStore()), cleanup: cacheCleanup}, + } + for _, tc := range tcs { + t.Run(tc.name, func(t *testing.T) { + defer tc.cleanup(file.DefaultDatabase, tc.s) + runStoreTest(tc.s, t) + }) + } + +} + +func TestStoreTable(t *testing.T) { + tcs := []struct { + name string + s store.Store + cleanup func(db string, s store.Store) + }{ + {name: "file", s: file.NewStore(store.Table("testTable")), cleanup: fileStoreCleanup}, + {name: "cockroach", s: cockroach.NewStore(store.Table("testTable")), cleanup: cockroachStoreCleanup}, + {name: "memory", s: memory.NewStore(store.Table("testTable")), cleanup: memoryCleanup}, + {name: "cache", s: cache.NewStore(memory.NewStore(store.Table("testTable"))), cleanup: cacheCleanup}, + } + for _, tc := range tcs { + t.Run(tc.name, func(t *testing.T) { + defer tc.cleanup(file.DefaultDatabase, tc.s) + runStoreTest(tc.s, t) + }) + } +} + +func TestStoreDatabase(t *testing.T) { + tcs := []struct { + name string + s store.Store + cleanup func(db string, s store.Store) + }{ + {name: "file", s: file.NewStore(store.Database("testdb")), cleanup: fileStoreCleanup}, + {name: "cockroach", s: cockroach.NewStore(store.Database("testdb")), cleanup: cockroachStoreCleanup}, + {name: "memory", s: memory.NewStore(store.Database("testdb")), cleanup: memoryCleanup}, + {name: "cache", s: cache.NewStore(memory.NewStore(store.Database("testdb"))), cleanup: cacheCleanup}, + } + for _, tc := range tcs { + t.Run(tc.name, func(t *testing.T) { + defer tc.cleanup("testdb", tc.s) + runStoreTest(tc.s, t) + }) + } +} + +func TestStoreDatabaseTable(t *testing.T) { + tcs := []struct { + name string + s store.Store + cleanup func(db string, s store.Store) + }{ + {name: "file", s: file.NewStore(store.Database("testdb"), store.Table("testTable")), cleanup: fileStoreCleanup}, + {name: "cockroach", s: cockroach.NewStore(store.Database("testdb"), store.Table("testTable")), cleanup: cockroachStoreCleanup}, + {name: "memory", s: memory.NewStore(store.Database("testdb"), store.Table("testTable")), cleanup: memoryCleanup}, + {name: "cache", s: cache.NewStore(memory.NewStore(store.Database("testdb"), store.Table("testTable"))), cleanup: cacheCleanup}, + } + for _, tc := range tcs { + t.Run(tc.name, func(t *testing.T) { + defer tc.cleanup("testdb", tc.s) + runStoreTest(tc.s, t) + }) + } +} + +func runStoreTest(s store.Store, t *testing.T) { + if len(os.Getenv("IN_TRAVIS_CI")) == 0 { + t.Logf("Options %s %v\n", s.String(), s.Options()) + } + + expiryTests(s, t) + suffixPrefixExpiryTests(s, t) + readTests(s, t) + listTests(s, t) + +} + +func readTests(s store.Store, t *testing.T) { + // Test Table, Suffix and WriteOptions + if err := s.Write(&store.Record{ + Key: "foofoobarbar", + Value: []byte("something"), + }, store.WriteTTL(time.Millisecond*100)); err != nil { + t.Error(err) + } + if err := s.Write(&store.Record{ + Key: "foofoo", + Value: []byte("something"), + }, store.WriteExpiry(time.Now().Add(time.Millisecond*100))); err != nil { + t.Error(err) + } + if err := s.Write(&store.Record{ + Key: "barbar", + Value: []byte("something"), + // TTL has higher precedence than expiry + }, store.WriteExpiry(time.Now().Add(time.Hour)), store.WriteTTL(time.Millisecond*100)); err != nil { + t.Error(err) + } + + if results, err := s.Read("foo", store.ReadPrefix(), store.ReadSuffix()); err != nil { + t.Error(err) + } else { + if len(results) != 1 { + t.Errorf("Expected 1 results, got %d: %# v", len(results), spew.Sdump(results)) + } + } + + time.Sleep(time.Millisecond * 100) + + if results, err := s.List(); err != nil { + t.Fatalf("List failed: %s", err) + } else { + if len(results) != 0 { + t.Fatalf("Expiry options were not effective, results :%v", spew.Sdump(results)) + } + } + + // write the following records + for i := 0; i < 10; i++ { + s.Write(&store.Record{ + Key: fmt.Sprintf("a%d", i), + Value: []byte{}, + }) + } + + // read back a few records + if results, err := s.Read("a", store.ReadLimit(5), store.ReadPrefix()); err != nil { + t.Error(err) + } else { + if len(results) != 5 { + t.Fatal("Expected 5 results, got ", len(results)) + } + if !strings.HasPrefix(results[0].Key, "a") { + t.Fatalf("Expected a prefix, got %s", results[0].Key) + } + } + + // read the rest back + if results, err := s.Read("a", store.ReadLimit(30), store.ReadOffset(5), store.ReadPrefix()); err != nil { + t.Fatal(err) + } else { + if len(results) != 5 { + t.Fatal("Expected 5 results, got ", len(results)) + } + } +} + +func listTests(s store.Store, t *testing.T) { + for i := 0; i < 10; i++ { + s.Write(&store.Record{Key: fmt.Sprintf("List%d", i), Value: []byte("bar")}) + } + + recs, err := s.List(store.ListPrefix("List")) + if err != nil { + t.Fatalf("Error listing records %s", err) + } + if len(recs) != 10 { + t.Fatalf("Expected 10 records, received %d", len(recs)) + } + + recs, err = s.List(store.ListPrefix("List"), store.ListLimit(5)) + if err != nil { + t.Fatalf("Error listing records %s", err) + } + if len(recs) != 5 { + t.Fatalf("Expected 5 records, received %d", len(recs)) + } + + recs, err = s.List(store.ListPrefix("List"), store.ListOffset(6)) + if err != nil { + t.Fatalf("Error listing records %s", err) + } + if len(recs) != 4 { + t.Fatalf("Expected 4 records, received %d %+v", len(recs), recs) + } + + recs, err = s.List(store.ListPrefix("List"), store.ListOffset(6), store.ListLimit(2)) + if err != nil { + t.Fatalf("Error listing records %s", err) + } + if len(recs) != 2 { + t.Fatalf("Expected 2 records, received %d %+v", len(recs), recs) + } + +} + +func expiryTests(s store.Store, t *testing.T) { + // Read and Write an expiring Record + if err := s.Write(&store.Record{ + Key: "Hello", + Value: []byte("World"), + Expiry: time.Millisecond * 150, + }); err != nil { + t.Error(err) + } + + if r, err := s.Read("Hello"); err != nil { + t.Fatal(err) + } else { + if len(r) != 1 { + t.Error("Read returned multiple records") + } + if r[0].Key != "Hello" { + t.Errorf("Expected %s, got %s", "Hello", r[0].Key) + } + if string(r[0].Value) != "World" { + t.Errorf("Expected %s, got %s", "World", r[0].Value) + } + } + + // wait for expiry + time.Sleep(time.Millisecond * 200) + + if _, err := s.Read("Hello"); err != store.ErrNotFound { + t.Errorf("Expected %# v, got %# v", store.ErrNotFound, err) + } + + // exercise the different ways to write an expiry, record.expiry + s.Write(&store.Record{Key: "aaa", Value: []byte("bbb"), Expiry: 1 * time.Second}) + s.Write(&store.Record{Key: "aaaa", Value: []byte("bbb"), Expiry: 1 * time.Second}) + s.Write(&store.Record{Key: "aaaaa", Value: []byte("bbb"), Expiry: 1 * time.Second}) + results, err := s.Read("a", store.ReadPrefix()) + if err != nil { + t.Error(err) + } + if len(results) != 3 { + t.Fatal("Results should have returned 3 records") + } + time.Sleep(1 * time.Second) + results, err = s.Read("a", store.ReadPrefix()) + if err != nil { + t.Error(err) + } + if len(results) != 0 { + t.Fatal("Results should have returned 0 records") + } + + // exercise the different ways to write an expiry, WriteExpiry + s.Write(&store.Record{Key: "bbb", Value: []byte("bbb")}, store.WriteExpiry(time.Now().Add(1*time.Second))) + s.Write(&store.Record{Key: "bbbb", Value: []byte("bbb")}, store.WriteExpiry(time.Now().Add(1*time.Second))) + s.Write(&store.Record{Key: "bbbbb", Value: []byte("bbb")}, store.WriteExpiry(time.Now().Add(1*time.Second))) + results, err = s.Read("b", store.ReadPrefix()) + if err != nil { + t.Error(err) + } + if len(results) != 3 { + t.Fatalf("Results should have returned 3 records. Received %d", len(results)) + } + time.Sleep(1 * time.Second) + results, err = s.Read("b", store.ReadPrefix()) + if err != nil { + t.Error(err) + } + if len(results) != 0 { + t.Fatalf("Results should have returned 0 records. Received %d", len(results)) + } + + // exercise the different ways to write an expiry, WriteTTL + s.Write(&store.Record{Key: "ccc", Value: []byte("bbb")}, store.WriteTTL(1*time.Second)) + s.Write(&store.Record{Key: "cccc", Value: []byte("bbb")}, store.WriteTTL(1*time.Second)) + s.Write(&store.Record{Key: "ccccc", Value: []byte("bbb")}, store.WriteTTL(1*time.Second)) + results, err = s.Read("c", store.ReadPrefix()) + if err != nil { + t.Error(err) + } + if len(results) != 3 { + t.Fatalf("Results should have returned 3 records. Received %d", len(results)) + } + time.Sleep(1 * time.Second) + results, err = s.Read("c", store.ReadPrefix()) + if err != nil { + t.Error(err) + } + if len(results) != 0 { + t.Fatalf("Results should have returned 0 records. Received %d", len(results)) + } +} + +func suffixPrefixExpiryTests(s store.Store, t *testing.T) { + // Write 3 records with various expiry and get with Prefix + records := []*store.Record{ + &store.Record{ + Key: "foo", + Value: []byte("foofoo"), + }, + &store.Record{ + Key: "foobar", + Value: []byte("foobarfoobar"), + Expiry: time.Millisecond * 100, + }, + } + + for _, r := range records { + if err := s.Write(r); err != nil { + t.Errorf("Couldn't write k: %s, v: %# v (%s)", r.Key, pretty.Formatter(r.Value), err) + } + } + + if results, err := s.Read("foo", store.ReadPrefix()); err != nil { + t.Errorf("Couldn't read all \"foo\" keys, got %#v (%s)", spew.Sdump(results), err) + } else { + if len(results) != 2 { + t.Errorf("Expected 2 items, got %d", len(results)) + } + } + + // wait for the expiry + time.Sleep(time.Millisecond * 200) + + if results, err := s.Read("foo", store.ReadPrefix()); err != nil { + t.Errorf("Couldn't read all \"foo\" keys, got %# v (%s)", spew.Sdump(results), err) + } else if len(results) != 1 { + t.Errorf("Expected 1 item, got %d", len(results)) + } + + if err := s.Delete("foo"); err != nil { + t.Errorf("Delete failed (%v)", err) + } + + if results, err := s.Read("foo"); err != store.ErrNotFound { + t.Errorf("Expected read failure read all \"foo\" keys, got %# v (%s)", spew.Sdump(results), err) + } else { + if len(results) != 0 { + t.Errorf("Expected 0 items, got %d (%# v)", len(results), spew.Sdump(results)) + } + } + + // Write 3 records with various expiry and get with Suffix + records = []*store.Record{ + &store.Record{ + Key: "foo", + Value: []byte("foofoo"), + }, + &store.Record{ + Key: "barfoo", + Value: []byte("barfoobarfoo"), + + Expiry: time.Millisecond * 100, + }, + &store.Record{ + Key: "bazbarfoo", + Value: []byte("bazbarfoobazbarfoo"), + Expiry: 2 * time.Millisecond * 100, + }, + } + for _, r := range records { + if err := s.Write(r); err != nil { + t.Errorf("Couldn't write k: %s, v: %# v (%s)", r.Key, pretty.Formatter(r.Value), err) + } + } + if results, err := s.Read("foo", store.ReadSuffix()); err != nil { + t.Errorf("Couldn't read all \"foo\" keys, got %# v (%s)", spew.Sdump(results), err) + } else { + if len(results) != 3 { + t.Errorf("Expected 3 items, got %d", len(results)) + //t.Logf("Table test: %v\n", spew.Sdump(results)) + } + + } + time.Sleep(time.Millisecond * 100) + if results, err := s.Read("foo", store.ReadSuffix()); err != nil { + t.Errorf("Couldn't read all \"foo\" keys, got %# v (%s)", spew.Sdump(results), err) + } else { + if len(results) != 2 { + t.Errorf("Expected 2 items, got %d", len(results)) + //t.Logf("Table test: %v\n", spew.Sdump(results)) + } + + } + time.Sleep(time.Millisecond * 100) + if results, err := s.Read("foo", store.ReadSuffix()); err != nil { + t.Errorf("Couldn't read all \"foo\" keys, got %# v (%s)", spew.Sdump(results), err) + } else { + if len(results) != 1 { + t.Errorf("Expected 1 item, got %d", len(results)) + // t.Logf("Table test: %# v\n", spew.Sdump(results)) + } + } + if err := s.Delete("foo"); err != nil { + t.Errorf("Delete failed (%v)", err) + } + if results, err := s.Read("foo", store.ReadSuffix()); err != nil { + t.Errorf("Couldn't read all \"foo\" keys, got %# v (%s)", spew.Sdump(results), err) + } else { + if len(results) != 0 { + t.Errorf("Expected 0 items, got %d (%# v)", len(results), spew.Sdump(results)) + } + } +}