diff --git a/api/server/acme/certmagic/certmagic.go b/api/server/acme/certmagic/certmagic.go index 81e485fc..0c62e4c7 100644 --- a/api/server/acme/certmagic/certmagic.go +++ b/api/server/acme/certmagic/certmagic.go @@ -48,7 +48,7 @@ func (c *certmagicProvider) TLSConfig(hosts ...string) (*tls.Config, error) { return certmagic.TLS(hosts) } -// New returns a certmagic provider +// NewProvider returns a certmagic provider func NewProvider(options ...acme.Option) acme.Provider { opts := acme.DefaultOptions() diff --git a/api/server/acme/certmagic/storage.go b/api/server/acme/certmagic/storage.go index 204269e6..dc805e98 100644 --- a/api/server/acme/certmagic/storage.go +++ b/api/server/acme/certmagic/storage.go @@ -88,16 +88,16 @@ func (s *storage) Exists(key string) bool { } func (s *storage) List(prefix string, recursive bool) ([]string, error) { - records, err := s.store.List() + keys, err := s.store.List() if err != nil { return nil, err } //nolint:prealloc var results []string - for _, r := range records { - if strings.HasPrefix(r.Key, prefix) { - results = append(results, r.Key) + for _, k := range keys { + if strings.HasPrefix(k, prefix) { + results = append(results, k) } } if recursive { diff --git a/go.mod b/go.mod index cb4fbcb2..1395da12 100644 --- a/go.mod +++ b/go.mod @@ -13,6 +13,7 @@ require ( github.com/coreos/go-semver v0.3.0 // indirect github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f // indirect github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f // indirect + github.com/cpuguy83/go-md2man/v2 v2.0.0 // indirect github.com/dgrijalva/jwt-go v3.2.0+incompatible github.com/forestgiant/sliceutil v0.0.0-20160425183142-94783f95db6c github.com/fsnotify/fsnotify v1.4.7 @@ -27,10 +28,8 @@ require ( github.com/gorilla/handlers v1.4.2 github.com/gorilla/websocket v1.4.1 github.com/grpc-ecosystem/go-grpc-middleware v1.1.0 // indirect - github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect github.com/hashicorp/hcl v1.0.0 github.com/imdario/mergo v0.3.8 - github.com/jonboulle/clockwork v0.1.0 // indirect github.com/joncalhoun/qson v0.0.0-20170526102502-8a9cab3a62b1 github.com/json-iterator/go v1.1.9 github.com/kr/pretty v0.1.0 @@ -48,12 +47,10 @@ require ( github.com/oxtoacart/bpool v0.0.0-20190530202638-03653db5a59c github.com/patrickmn/go-cache v2.1.0+incompatible github.com/pkg/errors v0.9.1 - github.com/soheilhy/cmux v0.1.4 // indirect github.com/stretchr/testify v1.4.0 github.com/technoweenie/multipartstreamer v1.0.1 // indirect github.com/tmc/grpc-websocket-proxy v0.0.0-20200122045848-3419fae592fc // indirect - github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 // indirect - go.etcd.io/bbolt v1.3.3 // indirect + go.etcd.io/etcd v0.5.0-alpha.5.0.20191031170918-4388404f56cb go.uber.org/zap v1.13.0 golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d golang.org/x/net v0.0.0-20200222125558-5a598a2470a0 @@ -63,5 +60,4 @@ require ( gopkg.in/go-playground/validator.v9 v9.31.0 gopkg.in/src-d/go-git.v4 v4.13.1 gopkg.in/telegram-bot-api.v4 v4.6.4 - sigs.k8s.io/yaml v1.1.0 // indirect ) diff --git a/go.sum b/go.sum index 17c35f27..ffc6da3e 100644 --- a/go.sum +++ b/go.sum @@ -59,6 +59,7 @@ github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24 github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= +github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bitly/go-simplejson v0.5.0 h1:6IH+V8/tVMab511d5bn4M7EwGXZf9Hj6i2xSwkNEM+Y= github.com/bitly/go-simplejson v0.5.0/go.mod h1:cXHtHw4XUPsvGaxgjIAn8PhEWG9NfngEKAMDJEczWVA= github.com/blang/semver v3.1.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= @@ -75,6 +76,8 @@ github.com/cheekybits/genny v1.0.0/go.mod h1:+tQajlRqAUrPI7DOSpB0XAqZYtQakVtB7wX github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= github.com/cloudflare/cloudflare-go v0.10.2 h1:VBodKICVPnwmDxstcW3biKcDSpFIfS/RELUXsZSBYK4= github.com/cloudflare/cloudflare-go v0.10.2/go.mod h1:qhVI5MKwBGhdNU89ZRz2plgYutcJ5PCekLxXn56w6SY= +github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa h1:OaNxuTZr7kxeODyLWsRMC+OD03aFUH+mW6r2d+MWa5Y= +github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/containerd/cgroups v0.0.0-20190919134610-bf292b21730f/go.mod h1:OApqhQ4XNSNC13gXIwDjhOQxjWa/NxkwZXJ1EvqT0ko= github.com/containerd/console v0.0.0-20180822173158-c12b1e7919c1/go.mod h1:Tj/on1eG8kiEhd0+fhSDzsPAFESxzBBvdyEgyryXffw= github.com/containerd/containerd v1.3.0-beta.2.0.20190828155532-0293cbd26c69/go.mod h1:bC6axHOhabU15QhwfG7w5PipXdVtMXFTttgp+kVtyUA= @@ -90,17 +93,22 @@ github.com/coreos/bbolt v1.3.3 h1:n6AiVyVRKQFNb6mJlwESEvvLoDyiTzXX7ORAUlkeBdY= github.com/coreos/bbolt v1.3.3/go.mod h1:iRUV2dpdMOn7Bo10OQBFzIJO9kkE559Wcmn+qkEiiKk= github.com/coreos/etcd v3.3.18+incompatible h1:Zz1aXgDrFFi1nadh58tA9ktt06cmPTwNNP3dXwIq1lE= github.com/coreos/etcd v3.3.18+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= +github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM= github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= +github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e h1:Wf6HqHfScWJN9/ZjdUKyjop4mf3Qdd+1TvvltAvM3m8= github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f h1:JOrtw2xFKzlg+cbHpyrpLDmnN1HqhBfnX7WDiW7eG2c= github.com/coreos/go-systemd v0.0.0-20190719114852-fd7a80b32e1f/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= +github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f h1:lBNOc5arjvs8E5mO2tbpBpLoyyu8B6e44T7hJy6potg= github.com/coreos/pkg v0.0.0-20180928190104-399ea9e2e55f/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= github.com/cpu/goacmedns v0.0.1/go.mod h1:sesf/pNnCYwUevQEQfEwY0Y3DydlQWSGZbaMElOWxok= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d h1:U+s90UTSYgptZMwQh2aRr3LuazLJIa+Pg3Kc1ylSYVY= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= +github.com/cpuguy83/go-md2man/v2 v2.0.0 h1:EoUDS0afbrsXAZ9YQ9jdu/mZ2sXgT1/2yyNng4PGlyM= +github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= @@ -118,6 +126,8 @@ github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKoh github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4 h1:qk/FSDDxo05wdJH28W+p5yivv7LuLYLRXPPD8KQCtZs= +github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= @@ -126,6 +136,7 @@ github.com/emirpasic/gods v1.12.0/go.mod h1:YfzfFFoVP/catgzJb4IKIqXjX78Ha8FMSDh3 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/exoscale/egoscale v0.18.1/go.mod h1:Z7OOdzzTOz1Q1PjQXumlz9Wn/CddH0zSYdCF3rnBKXE= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/structs v1.1.0/go.mod h1:9NiDSp5zOcgEDl+j00MP/WkGVPOlPRLejGD8Ga6PJ7M= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568 h1:BHsljHzVlRcyQhjrss6TZTdY2VfCqZPbv5k3iBFa2ZQ= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= @@ -164,6 +175,7 @@ github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zV github.com/goji/httpauth v0.0.0-20160601135302-2da839ab0f4d/go.mod h1:nnjvkQ9ptGaCkuDUx6wNykzzlUixGxvkme+H/lnzb+A= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= +github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e h1:1r7pUrabqp18hOBcwBwiTsbnFeTZHV9eER/QT5JVZxY= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= @@ -191,6 +203,7 @@ github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXi github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/pprof v0.0.0-20190515194954-54271f7e092f/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= @@ -203,16 +216,20 @@ github.com/gorilla/handlers v1.4.2/go.mod h1:Qkdc/uu4tH4g6mTK6auzZ766c4CA0Ng8+o/ github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/mux v1.7.3 h1:gnP5JzjVOuiZD07fKKToCAOjS0yOpj/qPETTXCCS6hw= github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= +github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.2.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.1 h1:q7AeDBpnBk8AogcD4DSag/Ukw/KV+YhzLj2bP5HvKCM= github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= github.com/grpc-ecosystem/go-grpc-middleware v1.1.0 h1:THDBEeQ9xZ8JEaCLyLQqXMMdRqNr0QAUJTIkQAUtFjg= github.com/grpc-ecosystem/go-grpc-middleware v1.1.0/go.mod h1:f5nM7jw/oeRSadq3xCzHAvxcr8HZnzsqU6ILg/0NiiE= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.8.5 h1:2+KSC78XiO6Qy0hIjfc1OD9H+hsaJdJlb8Kqsd41CTE= github.com/grpc-ecosystem/grpc-gateway v1.8.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= +github.com/grpc-ecosystem/grpc-gateway v1.9.5 h1:UImYN5qQ8tuGpGE16ZmjvcTtTw24zw1QAp/SlnNrZhI= +github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/h2non/parth v0.0.0-20190131123155-b4df798d6542/go.mod h1:Ow0tF8D4Kplbc8s8sSb3V2oUCygFHVp8gC3Dn6U4MNI= github.com/hashicorp/errwrap v0.0.0-20141028054710-7554cd9344ce/go.mod h1:YH+1FKiLXxHSkmPseP+kNlulaMuP3n2brvKWEqk/Jc4= github.com/hashicorp/go-multierror v0.0.0-20161216184304-ed905158d874/go.mod h1:JMRHfdO9jKNzS/+BTlxCjKNQHg/jZAft8U7LloJvN7I= @@ -226,6 +243,7 @@ github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpO github.com/iij/doapi v0.0.0-20190504054126-0bbf12d6d7df/go.mod h1:QMZY7/J/KSQEhKWFeDesPjMj+wCHReeknARU3wqlyN4= github.com/imdario/mergo v0.3.8 h1:CGgOkSJeqMRmt0D9XLWExdT4m4F1vd3FV3VPt+0VxkQ= github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99 h1:BQSFePA1RWJOlocH6Fxy8MmwDt+yVQYULKfN0RoTN8A= github.com/jbenet/go-context v0.0.0-20150711004518-d14ea06fba99/go.mod h1:1lJo3i6rXxKeerYnT8Nvf0QmHCRC1n8sfWVwXF2Frvo= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= @@ -274,7 +292,9 @@ github.com/marten-seemann/chacha20 v0.2.0/go.mod h1:HSdjFau7GzYRj+ahFNwsO3ouVJr1 github.com/marten-seemann/qpack v0.1.0/go.mod h1:LFt1NU/Ptjip0C2CPkhimBz5CGE3WGDAUWqna+CNTrI= github.com/marten-seemann/qtls v0.4.1 h1:YlT8QP3WCCvvok7MGEZkMldXbyqgr8oFg5/n8Gtbkks= github.com/marten-seemann/qtls v0.4.1/go.mod h1:pxVXcHHw1pNIt8Qo0pwSYQEoZ8yYOOPXTCZLQQunvRc= +github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-tty v0.0.0-20180219170247-931426f7535a/go.mod h1:XPvLUNfbS4fJH25nqRHfWLMa1ONC8Amw+mIA639KxkE= @@ -325,6 +345,7 @@ github.com/nrdcg/auroradns v1.0.0/go.mod h1:6JPXKzIRzZzMqtTDgueIhTi6rFf1QvYE/Hzq github.com/nrdcg/dnspod-go v0.3.0/go.mod h1:vZSoFSFeQVm2gWLMkyX61LZ8HI3BaqtHZWgPTGKr6KQ= github.com/nrdcg/goinwx v0.6.1/go.mod h1:XPiut7enlbEdntAqalBIqcYcTEVhpv/dKWgDCX2SwKQ= github.com/nrdcg/namesilo v0.2.1/go.mod h1:lwMvfQTyYq+BbjJd30ylEG4GPSS6PII0Tia4rRpRiyw= +github.com/olekukonko/tablewriter v0.0.0-20170122224234-a0225b3f23b5/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/olekukonko/tablewriter v0.0.1/go.mod h1:vsDQFd/mU46D+Z4whnwzcISnGGzXWMclvtLoiIKAKIo= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0 h1:WSHQ+IS43OoUrWtD1/bbclrwK8TTH5hzp+umCiuxHgs= @@ -398,6 +419,9 @@ github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1 github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/soheilhy/cmux v0.1.4 h1:0HKaf1o97UwFjHH9o5XsHUOF+tqmdA7KEzXLpiyaw0E= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= +github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= +github.com/spf13/pflag v1.0.1 h1:aCvUg6QPl3ibpQUxyLkrEkCHtPqYJL4x9AuhqVqFis4= +github.com/spf13/pflag v1.0.1/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/src-d/gcfg v1.4.0 h1:xXbNR5AlLSA315x2UO+fTSSAXCDf+Ar38/6oyGbDKQ4= github.com/src-d/gcfg v1.4.0/go.mod h1:p/UMsR43ujA89BJY9duynAwIpvqEujIH/jFlfL7jWoI= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -411,11 +435,13 @@ github.com/syndtr/gocapability v0.0.0-20170704070218-db04d3cc01c8/go.mod h1:hkRG github.com/technoweenie/multipartstreamer v1.0.1 h1:XRztA5MXiR1TIRHxH2uNxXxaIkKQDeX7m2XsSOlQEnM= github.com/technoweenie/multipartstreamer v1.0.1/go.mod h1:jNVxdtShOxzAsukZwTSw6MDx5eUJoiEBsSvzDU9uzog= github.com/timewasted/linode v0.0.0-20160829202747-37e84520dcf7/go.mod h1:imsgLplxEC/etjIhdr3dNzV3JeT27LbVu5pYWm0JCBY= +github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/tmc/grpc-websocket-proxy v0.0.0-20200122045848-3419fae592fc h1:yUaosFVTJwnltaHbSNC3i82I92quFs+OFPRl8kNMVwo= github.com/tmc/grpc-websocket-proxy v0.0.0-20200122045848-3419fae592fc/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/transip/gotransip v0.0.0-20190812104329-6d8d9179b66f/go.mod h1:i0f4R4o2HM0m3DZYQWsj6/MEowD57VzoH0v3d7igeFY= github.com/uber-go/atomic v1.3.2/go.mod h1:/Ct5t2lcmbJ4OSe/waGBoaVvVqtO0bmtfVNex1PFV8g= github.com/urfave/cli v0.0.0-20171014202726-7bc6a0acffa5/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= +github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/urfave/cli v1.22.1/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/vultr/govultr v0.1.4/go.mod h1:9H008Uxr/C4vFNGLqKx232C206GL0PBHzOP0809bGNA= github.com/xanzy/ssh-agent v0.2.1 h1:TCbipTQL2JiiCprBWx9frJ2eJlCYT00NmctrHxVAr70= @@ -428,6 +454,8 @@ github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2 h1:eY9dn8+vbi4tKz5 github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= go.etcd.io/bbolt v1.3.3 h1:MUGmc65QhB3pIlaQ5bB4LwqSj6GIonVJXpZiaKNyaKk= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.etcd.io/etcd v0.5.0-alpha.5.0.20191031170918-4388404f56cb h1:OIEw2droD31lXVboIQzJFUuSJOxGFReUW6hmqTP10TE= +go.etcd.io/etcd v0.5.0-alpha.5.0.20191031170918-4388404f56cb/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.20.2/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= go.opencensus.io v0.21.0/go.mod h1:mSImk1erAIZhrmZN+AvHh14ztQfjbGwt4TtuofqLduU= @@ -501,6 +529,7 @@ golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190930134127-c5a3c61f89f3/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20191027093000-83d349e8ac1a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -538,6 +567,7 @@ golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190801041406-cbf593c0f2f3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190826190057-c7b8b68b1456/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190904154756-749cb33beabd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190916202348-b4ddaad3f8a3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe h1:6fAMxZRR6sl1Uq8U61gxU+kPTs2tR8uOySCbBP7BN/M= @@ -548,6 +578,7 @@ golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190921001708-c4c64cad1fd0 h1:xQwXv67TxFo9nC1GJFyab5eq/5B590r6RlnL/G8Sz7w= @@ -608,12 +639,14 @@ google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiq google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= google.golang.org/grpc v1.22.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.23.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= +google.golang.org/grpc v1.23.1/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= google.golang.org/grpc v1.26.0 h1:2dTRdpdFEEhJYQD8EMLB61nnrzSCTbG38PhqdhvOltg= google.golang.org/grpc v1.26.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/cheggaaa/pb.v1 v1.0.25/go.mod h1:V/YB90LKu/1FcN3WVnfiiE5oMCibMjukxqG/qStrOgw= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= diff --git a/service.go b/service.go index a6c77469..9f5b6344 100644 --- a/service.go +++ b/service.go @@ -17,6 +17,7 @@ import ( "github.com/micro/go-micro/v2/logger" "github.com/micro/go-micro/v2/plugin" "github.com/micro/go-micro/v2/server" + "github.com/micro/go-micro/v2/store" "github.com/micro/go-micro/v2/util/config" "github.com/micro/go-micro/v2/util/wrapper" ) @@ -104,6 +105,13 @@ func (s *service) Init(opts ...Option) { logger.Fatal(err) } + // If the store has no namespace set, fallback to the + // services name + if len(store.DefaultStore.Options().Namespace) == 0 { + name := s.opts.Cmd.App().Name + store.DefaultStore.Init(store.Namespace(name)) + } + // TODO: replace Cmd.Init with config.Load // Right now we're just going to load a token // May need to re-read value on change diff --git a/store/cache/cache.go b/store/cache/cache.go new file mode 100644 index 00000000..1bb3852e --- /dev/null +++ b/store/cache/cache.go @@ -0,0 +1,39 @@ +package cache + +import ( + "github.com/micro/go-micro/v2/store" + "github.com/pkg/errors" +) + +// Cache implements a cache in front of a micro Store +type Cache struct { + options store.Options + store.Store + + stores []store.Store +} + +// NewStore returns new cache +func NewStore(opts ...store.Option) store.Store { + s := &Cache{ + options: store.Options{}, + stores: []store.Store{}, + } + for _, o := range opts { + o(&s.options) + } + return s +} + +// Init initialises a new cache +func (c *Cache) Init(opts ...store.Option) error { + for _, o := range opts { + o(&c.options) + } + for _, s := range c.stores { + if err := s.Init(); err != nil { + return errors.Wrapf(err, "Store %s failed to Init()", s.String()) + } + } + return nil +} diff --git a/store/cache/cache_test.go b/store/cache/cache_test.go new file mode 100644 index 00000000..bf01b9aa --- /dev/null +++ b/store/cache/cache_test.go @@ -0,0 +1,15 @@ +package cache + +// import "testing" + +// func TestCache(t *testing.T) { +// c := NewStore() +// if err := c.Init(); err != nil { +// //t.Fatal(err) +// } +// if results, err := c.Read("test"); err != nil { +// //t.Fatal(err) +// } else { +// println(results) +// } +// } diff --git a/store/cloudflare/cloudflare.go b/store/cloudflare/cloudflare.go index b6177c66..046276fa 100644 --- a/store/cloudflare/cloudflare.go +++ b/store/cloudflare/cloudflare.go @@ -159,25 +159,13 @@ func (w *workersKV) list(prefix string) ([]string, error) { // In the cloudflare workers KV implemention, List() doesn't guarantee // anything as the workers API is eventually consistent. -func (w *workersKV) List() ([]*store.Record, error) { +func (w *workersKV) List(opts ...store.ListOption) ([]string, error) { keys, err := w.list("") if err != nil { return nil, err } - var gerr error - var records []*store.Record - - for _, key := range keys { - r, err := w.Read(key) - if err != nil { - gerr = err - continue - } - records = append(records, r...) - } - - return records, gerr + return keys, nil } func (w *workersKV) Read(key string, opts ...store.ReadOption) ([]*store.Record, error) { @@ -244,7 +232,7 @@ func (w *workersKV) Read(key string, opts ...store.ReadOption) ([]*store.Record, return records, nil } -func (w *workersKV) Write(r *store.Record) error { +func (w *workersKV) Write(r *store.Record, opts ...store.WriteOption) error { // Set it in local cache, with the global TTL from options if w.cache != nil { w.cache.Set(r.Key, r, cache.DefaultExpiration) @@ -282,7 +270,7 @@ func (w *workersKV) Write(r *store.Record) error { return nil } -func (w *workersKV) Delete(key string) error { +func (w *workersKV) Delete(key string, opts ...store.DeleteOption) error { if w.cache != nil { w.cache.Delete(key) } @@ -371,6 +359,10 @@ func (w *workersKV) String() string { return "cloudflare" } +func (w *workersKV) Options() store.Options { + return w.options +} + // NewStore returns a cloudflare Store implementation. // Account ID, Token and Namespace must either be passed as options or // environment variables. If set as env vars we expect the following; diff --git a/store/cockroach/cockroach.go b/store/cockroach/cockroach.go index fcda1d6f..4bd92576 100644 --- a/store/cockroach/cockroach.go +++ b/store/cockroach/cockroach.go @@ -28,6 +28,11 @@ type sqlStore struct { database string table string + list *sql.Stmt + readOne *sql.Stmt + write *sql.Stmt + delete *sql.Stmt + options store.Options } @@ -40,13 +45,13 @@ func (s *sqlStore) Init(opts ...store.Option) error { } // List all the known records -func (s *sqlStore) List() ([]*store.Record, error) { - rows, err := s.db.Query(fmt.Sprintf("SELECT key, value, expiry FROM %s.%s;", s.database, s.table)) - var records []*store.Record +func (s *sqlStore) List(opts ...store.ListOption) ([]string, error) { + rows, err := s.list.Query() + var keys []string var timehelper pq.NullTime if err != nil { if err == sql.ErrNoRows { - return records, nil + return keys, nil } return nil, err } @@ -54,7 +59,7 @@ func (s *sqlStore) List() ([]*store.Record, error) { for rows.Next() { record := &store.Record{} if err := rows.Scan(&record.Key, &record.Value, &timehelper); err != nil { - return records, err + return keys, err } if timehelper.Valid { if timehelper.Time.Before(time.Now()) { @@ -62,25 +67,25 @@ func (s *sqlStore) List() ([]*store.Record, error) { go s.Delete(record.Key) } else { record.Expiry = time.Until(timehelper.Time) - records = append(records, record) + keys = append(keys, record.Key) } } else { - records = append(records, record) + keys = append(keys, record.Key) } } rowErr := rows.Close() if rowErr != nil { // transaction rollback or something - return records, rowErr + return keys, rowErr } if err := rows.Err(); err != nil { - return records, err + return keys, err } - return records, nil + return keys, nil } -// Read all records with keys +// Read a single key func (s *sqlStore) Read(key string, opts ...store.ReadOption) ([]*store.Record, error) { var options store.ReadOptions for _, o := range opts { @@ -89,15 +94,10 @@ func (s *sqlStore) Read(key string, opts ...store.ReadOption) ([]*store.Record, // TODO: make use of options.Prefix using WHERE key LIKE = ? - q, err := s.db.Prepare(fmt.Sprintf("SELECT key, value, expiry FROM %s.%s WHERE key = $1;", s.database, s.table)) - if err != nil { - return nil, err - } - var records []*store.Record var timehelper pq.NullTime - row := q.QueryRow(key) + row := s.readOne.QueryRow(key) record := &store.Record{} if err := row.Scan(&record.Key, &record.Value, &timehelper); err != nil { if err == sql.ErrNoRows { @@ -121,20 +121,12 @@ func (s *sqlStore) Read(key string, opts ...store.ReadOption) ([]*store.Record, } // Write records -func (s *sqlStore) Write(r *store.Record) error { - q, err := s.db.Prepare(fmt.Sprintf(`INSERT INTO %s.%s(key, value, expiry) - VALUES ($1, $2::bytea, $3) - ON CONFLICT (key) - DO UPDATE - SET value = EXCLUDED.value, expiry = EXCLUDED.expiry;`, s.database, s.table)) - if err != nil { - return err - } - +func (s *sqlStore) Write(r *store.Record, opts ...store.WriteOption) error { + var err error if r.Expiry != 0 { - _, err = q.Exec(r.Key, r.Value, time.Now().Add(r.Expiry)) + _, err = s.write.Exec(r.Key, r.Value, time.Now().Add(r.Expiry)) } else { - _, err = q.Exec(r.Key, r.Value, nil) + _, err = s.write.Exec(r.Key, r.Value, nil) } if err != nil { @@ -145,13 +137,8 @@ func (s *sqlStore) Write(r *store.Record) error { } // Delete records with keys -func (s *sqlStore) Delete(key string) error { - q, err := s.db.Prepare(fmt.Sprintf("DELETE FROM %s.%s WHERE key = $1;", s.database, s.table)) - if err != nil { - return err - } - - result, err := q.Exec(key) +func (s *sqlStore) Delete(key string, opts ...store.DeleteOption) error { + result, err := s.delete.Exec(key) if err != nil { return err } @@ -187,6 +174,31 @@ func (s *sqlStore) initDB() error { return errors.Wrap(err, "Couldn't create table") } + list, err := s.db.Prepare(fmt.Sprintf("SELECT key, value, expiry FROM %s.%s;", s.database, s.table)) + if err != nil { + return errors.Wrap(err, "List statement couldn't be prepared") + } + s.list = list + readOne, err := s.db.Prepare(fmt.Sprintf("SELECT key, value, expiry FROM %s.%s WHERE key = $1;", s.database, s.table)) + if err != nil { + return errors.Wrap(err, "ReadOne statement couldn't be prepared") + } + s.readOne = readOne + write, err := s.db.Prepare(fmt.Sprintf(`INSERT INTO %s.%s(key, value, expiry) + VALUES ($1, $2::bytea, $3) + ON CONFLICT (key) + DO UPDATE + SET value = EXCLUDED.value, expiry = EXCLUDED.expiry;`, s.database, s.table)) + if err != nil { + return errors.Wrap(err, "Write statement couldn't be prepared") + } + s.write = write + delete, err := s.db.Prepare(fmt.Sprintf("DELETE FROM %s.%s WHERE key = $1;", s.database, s.table)) + if err != nil { + return errors.Wrap(err, "Delete statement couldn't be prepared") + } + s.delete = delete + return nil } @@ -206,12 +218,12 @@ func (s *sqlStore) configure() error { prefix = DefaultPrefix } - // store.namespace must only contain letters + // store.namespace must only contain letters, numbers and underscores reg, err := regexp.Compile("[^a-zA-Z0-9]+") if err != nil { return errors.New("error compiling regex for namespace") } - namespace = reg.ReplaceAllString(namespace, "") + namespace = reg.ReplaceAllString(namespace, "_") source := nodes[0] // check if it is a standard connection string eg: host=%s port=%d user=%s password=%s dbname=%s sslmode=disable @@ -250,7 +262,11 @@ func (s *sqlStore) String() string { return "cockroach" } -// New returns a new micro Store backed by sql +func (s *sqlStore) Options() store.Options { + return s.options +} + +// NewStore returns a new micro Store backed by sql func NewStore(opts ...store.Option) store.Store { var options store.Options for _, o := range opts { diff --git a/store/cockroach/cockroach_test.go b/store/cockroach/cockroach_test.go index 02354eac..38e36f5f 100644 --- a/store/cockroach/cockroach_test.go +++ b/store/cockroach/cockroach_test.go @@ -32,11 +32,11 @@ func TestSQL(t *testing.T) { store.Nodes(connection), ) - records, err := sqlStore.List() + keys, err := sqlStore.List() if err != nil { t.Error(err) } else { - t.Logf("%# v\n", pretty.Formatter(records)) + t.Logf("%# v\n", pretty.Formatter(keys)) } err = sqlStore.Write( @@ -80,7 +80,7 @@ func TestSQL(t *testing.T) { t.Error(err) } - records, err = sqlStore.Read("test") + records, err := sqlStore.Read("test") if err != nil { t.Error(err) } diff --git a/store/etcd/config.go b/store/etcd/config.go new file mode 100644 index 00000000..381abdb3 --- /dev/null +++ b/store/etcd/config.go @@ -0,0 +1,178 @@ +package etcd + +import ( + "context" + cryptotls "crypto/tls" + "time" + + "github.com/micro/go-micro/v2/store" + "go.etcd.io/etcd/clientv3" + "google.golang.org/grpc" +) + +// Implement all the options from https://pkg.go.dev/go.etcd.io/etcd/clientv3?tab=doc#Config +// Need to use non basic types in context.WithValue +type autoSyncInterval string +type dialTimeout string +type dialKeepAliveTime string +type dialKeepAliveTimeout string +type maxCallSendMsgSize string +type maxCallRecvMsgSize string +type tls string +type username string +type password string +type rejectOldCluster string +type dialOptions string +type clientContext string +type permitWithoutStream string + +// AutoSyncInterval is the interval to update endpoints with its latest members. +// 0 disables auto-sync. By default auto-sync is disabled. +func AutoSyncInterval(d time.Duration) store.Option { + return func(o *store.Options) { + o.Context = context.WithValue(o.Context, autoSyncInterval(""), d) + } +} + +// DialTimeout is the timeout for failing to establish a connection. +func DialTimeout(d time.Duration) store.Option { + return func(o *store.Options) { + o.Context = context.WithValue(o.Context, dialTimeout(""), d) + } +} + +// DialKeepAliveTime is the time after which client pings the server to see if +// transport is alive. +func DialKeepAliveTime(d time.Duration) store.Option { + return func(o *store.Options) { + o.Context = context.WithValue(o.Context, dialKeepAliveTime(""), d) + } +} + +// DialKeepAliveTimeout is the time that the client waits for a response for the +// keep-alive probe. If the response is not received in this time, the connection is closed. +func DialKeepAliveTimeout(d time.Duration) store.Option { + return func(o *store.Options) { + o.Context = context.WithValue(o.Context, dialKeepAliveTimeout(""), d) + } +} + +// MaxCallSendMsgSize is the client-side request send limit in bytes. +// If 0, it defaults to 2.0 MiB (2 * 1024 * 1024). +// Make sure that "MaxCallSendMsgSize" < server-side default send/recv limit. +// ("--max-request-bytes" flag to etcd or "embed.Config.MaxRequestBytes"). +func MaxCallSendMsgSize(size int) store.Option { + return func(o *store.Options) { + o.Context = context.WithValue(o.Context, maxCallSendMsgSize(""), size) + } +} + +// MaxCallRecvMsgSize is the client-side response receive limit. +// If 0, it defaults to "math.MaxInt32", because range response can +// easily exceed request send limits. +// Make sure that "MaxCallRecvMsgSize" >= server-side default send/recv limit. +// ("--max-request-bytes" flag to etcd or "embed.Config.MaxRequestBytes"). +func MaxCallRecvMsgSize(size int) store.Option { + return func(o *store.Options) { + o.Context = context.WithValue(o.Context, maxCallRecvMsgSize(""), size) + } +} + +// TLS holds the client secure credentials, if any. +func TLS(conf *cryptotls.Config) store.Option { + return func(o *store.Options) { + t := conf.Clone() + o.Context = context.WithValue(o.Context, tls(""), t) + } +} + +// Username is a user name for authentication. +func Username(u string) store.Option { + return func(o *store.Options) { + o.Context = context.WithValue(o.Context, username(""), u) + } +} + +// Password is a password for authentication. +func Password(p string) store.Option { + return func(o *store.Options) { + o.Context = context.WithValue(o.Context, password(""), p) + } +} + +// RejectOldCluster when set will refuse to create a client against an outdated cluster. +func RejectOldCluster(b bool) store.Option { + return func(o *store.Options) { + o.Context = context.WithValue(o.Context, rejectOldCluster(""), b) + } +} + +// DialOptions is a list of dial options for the grpc client (e.g., for interceptors). +// For example, pass "grpc.WithBlock()" to block until the underlying connection is up. +// Without this, Dial returns immediately and connecting the server happens in background. +func DialOptions(opts []grpc.DialOption) store.Option { + return func(o *store.Options) { + if len(opts) > 0 { + ops := make([]grpc.DialOption, len(opts)) + copy(ops, opts) + o.Context = context.WithValue(o.Context, dialOptions(""), ops) + } + } +} + +// ClientContext is the default etcd3 client context; it can be used to cancel grpc +// dial out andother operations that do not have an explicit context. +func ClientContext(ctx context.Context) store.Option { + return func(o *store.Options) { + o.Context = context.WithValue(o.Context, clientContext(""), ctx) + } +} + +// PermitWithoutStream when set will allow client to send keepalive pings to server without any active streams(RPCs). +func PermitWithoutStream(b bool) store.Option { + return func(o *store.Options) { + o.Context = context.WithValue(o.Context, permitWithoutStream(""), b) + } +} + +func (e *etcdStore) applyConfig(cfg *clientv3.Config) { + if v := e.options.Context.Value(autoSyncInterval("")); v != nil { + cfg.AutoSyncInterval = v.(time.Duration) + } + if v := e.options.Context.Value(dialTimeout("")); v != nil { + cfg.DialTimeout = v.(time.Duration) + } + if v := e.options.Context.Value(dialKeepAliveTime("")); v != nil { + cfg.DialKeepAliveTime = v.(time.Duration) + } + if v := e.options.Context.Value(dialKeepAliveTimeout("")); v != nil { + cfg.DialKeepAliveTimeout = v.(time.Duration) + } + if v := e.options.Context.Value(maxCallSendMsgSize("")); v != nil { + cfg.MaxCallSendMsgSize = v.(int) + } + if v := e.options.Context.Value(maxCallRecvMsgSize("")); v != nil { + cfg.MaxCallRecvMsgSize = v.(int) + } + if v := e.options.Context.Value(tls("")); v != nil { + cfg.TLS = v.(*cryptotls.Config) + } + if v := e.options.Context.Value(username("")); v != nil { + cfg.Username = v.(string) + } + if v := e.options.Context.Value(password("")); v != nil { + cfg.Username = v.(string) + } + if v := e.options.Context.Value(rejectOldCluster("")); v != nil { + cfg.RejectOldCluster = v.(bool) + } + if v := e.options.Context.Value(dialOptions("")); v != nil { + cfg.DialOptions = v.([]grpc.DialOption) + } + if v := e.options.Context.Value(clientContext("")); v != nil { + cfg.Context = v.(context.Context) + } + if v := e.options.Context.Value(permitWithoutStream("")); v != nil { + cfg.PermitWithoutStream = v.(bool) + } +} diff --git a/store/etcd/etcd.go b/store/etcd/etcd.go index fd896c7a..2462c55d 100644 --- a/store/etcd/etcd.go +++ b/store/etcd/etcd.go @@ -1,117 +1,268 @@ -// Package etcd is an etcd v3 implementation of kv +// Package etcd implements a go-micro/v2/store with etcd package etcd import ( + "bytes" "context" - "log" + "encoding/gob" + "math" + "strings" + "time" - client "github.com/coreos/etcd/clientv3" "github.com/micro/go-micro/v2/store" + "github.com/pkg/errors" + "go.etcd.io/etcd/clientv3" + "go.etcd.io/etcd/clientv3/namespace" ) -type ekv struct { +type etcdStore struct { options store.Options - kv client.KV + + client *clientv3.Client + config clientv3.Config } -func (e *ekv) Init(opts ...store.Option) error { +// NewStore returns a new etcd store +func NewStore(opts ...store.Option) store.Store { + e := &etcdStore{} for _, o := range opts { o(&e.options) } + e.init() + return e +} + +func (e *etcdStore) Init(opts ...store.Option) error { + for _, o := range opts { + o(&e.options) + } + return e.init() +} + +func (e *etcdStore) init() error { + // ensure context is non-nil + e.options.Context = context.Background() + // set up config + e.config = clientv3.Config{} + e.applyConfig(&e.config) + if len(e.options.Nodes) == 0 { + e.config.Endpoints = []string{"http://127.0.0.1:2379"} + } else { + e.config.Endpoints = make([]string, len(e.options.Nodes)) + copy(e.config.Endpoints, e.options.Nodes) + } + if e.client != nil { + e.client.Close() + } + client, err := clientv3.New(e.config) + if err != nil { + return err + } + e.client = client + ns := "" + if len(e.options.Prefix) > 0 { + ns = e.options.Prefix + } + if len(e.options.Namespace) > 0 { + ns = e.options.Namespace + "/" + ns + } + if len(ns) > 0 { + e.client.KV = namespace.NewKV(e.client.KV, ns) + e.client.Watcher = namespace.NewWatcher(e.client.Watcher, ns) + e.client.Lease = namespace.NewLease(e.client.Lease, ns) + } + return nil } -func (e *ekv) Read(key string, opts ...store.ReadOption) ([]*store.Record, error) { - var options store.ReadOptions - for _, o := range opts { - o(&options) - } - - var etcdOpts []client.OpOption - - // set options prefix - if options.Prefix { - etcdOpts = append(etcdOpts, client.WithPrefix()) - } - - keyval, err := e.kv.Get(context.Background(), key, etcdOpts...) - if err != nil { - return nil, err - } - - if keyval == nil || len(keyval.Kvs) == 0 { - return nil, store.ErrNotFound - } - - records := make([]*store.Record, 0, len(keyval.Kvs)) - - for _, kv := range keyval.Kvs { - records = append(records, &store.Record{ - Key: string(kv.Key), - Value: kv.Value, - // TODO: implement expiry - }) - } - - return records, nil +func (e *etcdStore) Options() store.Options { + return e.options } -func (e *ekv) Delete(key string) error { - _, err := e.kv.Delete(context.Background(), key) - return err -} - -func (e *ekv) Write(record *store.Record) error { - // TODO create lease to expire keys - _, err := e.kv.Put(context.Background(), record.Key, string(record.Value)) - return err -} - -func (e *ekv) List() ([]*store.Record, error) { - keyval, err := e.kv.Get(context.Background(), "/", client.WithPrefix()) - if err != nil { - return nil, err - } - if keyval == nil || len(keyval.Kvs) == 0 { - return nil, nil - } - vals := make([]*store.Record, 0, len(keyval.Kvs)) - for _, keyv := range keyval.Kvs { - vals = append(vals, &store.Record{ - Key: string(keyv.Key), - Value: keyv.Value, - }) - } - return vals, nil -} - -func (e *ekv) String() string { +func (e *etcdStore) String() string { return "etcd" } -func NewStore(opts ...store.Option) store.Store { - var options store.Options +func (e *etcdStore) Read(key string, opts ...store.ReadOption) ([]*store.Record, error) { + readOpts := store.ReadOptions{} + for _, o := range opts { + o(&readOpts) + } + if readOpts.Suffix { + return e.readSuffix(key, readOpts) + } + + var etcdOpts []clientv3.OpOption + if readOpts.Prefix { + etcdOpts = append(etcdOpts, clientv3.WithPrefix(), clientv3.WithSort(clientv3.SortByKey, clientv3.SortAscend)) + } + resp, err := e.client.KV.Get(context.Background(), key, etcdOpts...) + if err != nil { + return nil, err + } + if resp.Count == 0 && !(readOpts.Prefix || readOpts.Suffix) { + return nil, store.ErrNotFound + } + var records []*store.Record + for _, kv := range resp.Kvs { + ir := internalRecord{} + if err := gob.NewDecoder(bytes.NewReader(kv.Value)).Decode(&ir); err != nil { + return records, errors.Wrapf(err, "couldn't decode %s into internalRecord", err.Error()) + } + r := store.Record{ + Key: ir.Key, + Value: ir.Value, + } + if !ir.ExpiresAt.IsZero() { + r.Expiry = time.Until(ir.ExpiresAt) + } + records = append(records, &r) + } + if readOpts.Limit > 0 || readOpts.Offset > 0 { + return records[readOpts.Offset:min(readOpts.Limit, uint(len(records)))], nil + } + return records, nil +} + +func (e *etcdStore) readSuffix(key string, readOpts store.ReadOptions) ([]*store.Record, error) { + opts := []store.ListOption{store.ListSuffix(key)} + if readOpts.Prefix { + opts = append(opts, store.ListPrefix(key)) + } + keys, err := e.List(opts...) + if err != nil { + return nil, errors.Wrapf(err, "Couldn't list with suffix %s", key) + } + var records []*store.Record + for _, k := range keys { + resp, err := e.client.KV.Get(context.Background(), k) + if err != nil { + return nil, errors.Wrapf(err, "Couldn't get key %s", k) + } + ir := internalRecord{} + if err := gob.NewDecoder(bytes.NewReader(resp.Kvs[0].Value)).Decode(&ir); err != nil { + return records, errors.Wrapf(err, "couldn't decode %s into internalRecord", err.Error()) + } + r := store.Record{ + Key: ir.Key, + Value: ir.Value, + } + if !ir.ExpiresAt.IsZero() { + r.Expiry = time.Until(ir.ExpiresAt) + } + records = append(records, &r) + + } + if readOpts.Limit > 0 || readOpts.Offset > 0 { + return records[readOpts.Offset:min(readOpts.Limit, uint(len(records)))], nil + } + return records, nil +} + +func (e *etcdStore) Write(r *store.Record, opts ...store.WriteOption) error { + options := store.WriteOptions{} for _, o := range opts { o(&options) } - // get the endpoints - endpoints := options.Nodes + if len(opts) > 0 { + // Copy the record before applying options, or the incoming record will be mutated + newRecord := store.Record{} + newRecord.Key = r.Key + newRecord.Value = make([]byte, len(r.Value)) + copy(newRecord.Value, r.Value) + newRecord.Expiry = r.Expiry - if len(endpoints) == 0 { - endpoints = []string{"http://127.0.0.1:2379"} - } - - // TODO: parse addresses - c, err := client.New(client.Config{ - Endpoints: endpoints, - }) - if err != nil { - log.Fatal(err) - } - - return &ekv{ - options: options, - kv: client.NewKV(c), + if !options.Expiry.IsZero() { + newRecord.Expiry = time.Until(options.Expiry) + } + if options.TTL != 0 { + newRecord.Expiry = options.TTL + } + return e.write(&newRecord) } + return e.write(r) +} + +func (e *etcdStore) write(r *store.Record) error { + var putOpts []clientv3.OpOption + ir := &internalRecord{} + ir.Key = r.Key + ir.Value = make([]byte, len(r.Value)) + copy(ir.Value, r.Value) + if r.Expiry != 0 { + ir.ExpiresAt = time.Now().Add(r.Expiry) + var leasexpiry int64 + if r.Expiry.Seconds() < 5.0 { + // minimum etcd lease is 5 seconds + leasexpiry = 5 + } else { + leasexpiry = int64(math.Ceil(r.Expiry.Seconds())) + } + lr, err := e.client.Lease.Grant(context.Background(), leasexpiry) + if err != nil { + return errors.Wrapf(err, "couldn't grant an etcd lease for %s", r.Key) + } + putOpts = append(putOpts, clientv3.WithLease(lr.ID)) + } + b := &bytes.Buffer{} + if err := gob.NewEncoder(b).Encode(ir); err != nil { + return errors.Wrapf(err, "couldn't encode %s", r.Key) + } + _, err := e.client.KV.Put(context.Background(), ir.Key, string(b.Bytes()), putOpts...) + return errors.Wrapf(err, "couldn't put key %s in to etcd", err) +} + +func (e *etcdStore) Delete(key string, opts ...store.DeleteOption) error { + options := store.DeleteOptions{} + for _, o := range opts { + o(&options) + } + _, err := e.client.KV.Delete(context.Background(), key) + return errors.Wrapf(err, "couldn't delete key %s", key) +} + +func (e *etcdStore) List(opts ...store.ListOption) ([]string, error) { + options := store.ListOptions{} + for _, o := range opts { + o(&options) + } + searchPrefix := "" + if len(options.Prefix) > 0 { + searchPrefix = options.Prefix + } + resp, err := e.client.KV.Get(context.Background(), searchPrefix, clientv3.WithPrefix(), clientv3.WithKeysOnly(), clientv3.WithSort(clientv3.SortByKey, clientv3.SortAscend)) + if err != nil { + return nil, errors.Wrap(err, "couldn't list, etcd get failed") + } + if len(options.Suffix) == 0 { + keys := make([]string, resp.Count) + for i, kv := range resp.Kvs { + keys[i] = string(kv.Key) + } + return keys, nil + } + keys := []string{} + for _, kv := range resp.Kvs { + if strings.HasSuffix(string(kv.Key), options.Suffix) { + keys = append(keys, string(kv.Key)) + } + } + if options.Limit > 0 || options.Offset > 0 { + return keys[options.Offset:min(options.Limit, uint(len(keys)))], nil + } + return keys, nil +} + +type internalRecord struct { + Key string + Value []byte + ExpiresAt time.Time +} + +func min(i, j uint) uint { + if i < j { + return i + } + return j } diff --git a/store/etcd/etcd_test.go b/store/etcd/etcd_test.go new file mode 100644 index 00000000..fade2fce --- /dev/null +++ b/store/etcd/etcd_test.go @@ -0,0 +1,225 @@ +package etcd + +import ( + "fmt" + "testing" + "time" + + "github.com/kr/pretty" + "github.com/micro/go-micro/v2/store" +) + +func TestEtcd(t *testing.T) { + e := NewStore() + if err := e.Init(); err != nil { + t.Fatal(err) + } + //basictest(e, t) +} + +func basictest(s store.Store, t *testing.T) { + t.Logf("Testing store %s, with options %# v\n", s.String(), pretty.Formatter(s.Options())) + // Read and Write an expiring Record + if err := s.Write(&store.Record{ + Key: "Hello", + Value: []byte("World"), + Expiry: time.Second * 5, + }); err != nil { + t.Fatal(err) + } + if r, err := s.Read("Hello"); err != nil { + t.Fatal(err) + } else { + if len(r) != 1 { + t.Fatal("Read returned multiple records") + } + if r[0].Key != "Hello" { + t.Fatalf("Expected %s, got %s", "Hello", r[0].Key) + } + if string(r[0].Value) != "World" { + t.Fatalf("Expected %s, got %s", "World", r[0].Value) + } + } + time.Sleep(time.Second * 6) + if records, err := s.Read("Hello"); err != store.ErrNotFound { + t.Fatalf("Expected %# v, got %# v\nResults were %# v", store.ErrNotFound, err, pretty.Formatter(records)) + } + + // Write 3 records with various expiry and get with prefix + records := []*store.Record{ + &store.Record{ + Key: "foo", + Value: []byte("foofoo"), + }, + &store.Record{ + Key: "foobar", + Value: []byte("foobarfoobar"), + Expiry: time.Second * 5, + }, + &store.Record{ + Key: "foobarbaz", + Value: []byte("foobarbazfoobarbaz"), + Expiry: 2 * time.Second * 5, + }, + } + for _, r := range records { + if err := s.Write(r); err != nil { + t.Fatalf("Couldn't write k: %s, v: %# v (%s)", r.Key, pretty.Formatter(r.Value), err) + } + } + if results, err := s.Read("foo", store.ReadPrefix()); err != nil { + t.Fatalf("Couldn't read all \"foo\" keys, got %# v (%s)", pretty.Formatter(results), err) + } else { + if len(results) != 3 { + t.Fatalf("Expected 3 items, got %d", len(results)) + } + } + time.Sleep(time.Second * 6) + if results, err := s.Read("foo", store.ReadPrefix()); err != nil { + t.Fatalf("Couldn't read all \"foo\" keys, got %# v (%s)", pretty.Formatter(results), err) + } else { + if len(results) != 2 { + t.Fatalf("Expected 2 items, got %d", len(results)) + } + } + time.Sleep(time.Second * 5) + if results, err := s.Read("foo", store.ReadPrefix()); err != nil { + t.Fatalf("Couldn't read all \"foo\" keys, got %# v (%s)", pretty.Formatter(results), err) + } else { + if len(results) != 1 { + t.Fatalf("Expected 1 item, got %d", len(results)) + } + } + if err := s.Delete("foo", func(d *store.DeleteOptions) {}); err != nil { + t.Fatalf("Delete failed (%v)", err) + } + if results, err := s.Read("foo", store.ReadPrefix()); err != nil { + t.Fatalf("Couldn't read all \"foo\" keys, got %# v (%s)", pretty.Formatter(results), err) + } else { + if len(results) != 0 { + t.Fatalf("Expected 0 items, got %d (%# v)", len(results), pretty.Formatter(results)) + } + } + + // Write 3 records with various expiry and get with Suffix + records = []*store.Record{ + &store.Record{ + Key: "foo", + Value: []byte("foofoo"), + }, + &store.Record{ + Key: "barfoo", + Value: []byte("barfoobarfoo"), + Expiry: time.Second * 5, + }, + &store.Record{ + Key: "bazbarfoo", + Value: []byte("bazbarfoobazbarfoo"), + Expiry: 2 * time.Second * 5, + }, + } + for _, r := range records { + if err := s.Write(r); err != nil { + t.Fatalf("Couldn't write k: %s, v: %# v (%s)", r.Key, pretty.Formatter(r.Value), err) + } + } + if results, err := s.Read("foo", store.ReadSuffix()); err != nil { + t.Fatalf("Couldn't read all \"foo\" keys, got %# v (%s)", pretty.Formatter(results), err) + } else { + if len(results) != 3 { + t.Fatalf("Expected 3 items, got %d", len(results)) + } + } + time.Sleep(time.Second * 6) + if results, err := s.Read("foo", store.ReadSuffix()); err != nil { + t.Fatalf("Couldn't read all \"foo\" keys, got %# v (%s)", pretty.Formatter(results), err) + } else { + if len(results) != 2 { + t.Fatalf("Expected 2 items, got %d", len(results)) + } + t.Logf("Prefix test: %v\n", pretty.Formatter(results)) + } + time.Sleep(time.Second * 5) + if results, err := s.Read("foo", store.ReadSuffix()); err != nil { + t.Fatalf("Couldn't read all \"foo\" keys, got %# v (%s)", pretty.Formatter(results), err) + } else { + if len(results) != 1 { + t.Fatalf("Expected 1 item, got %d", len(results)) + } + t.Logf("Prefix test: %# v\n", pretty.Formatter(results)) + } + if err := s.Delete("foo"); err != nil { + t.Fatalf("Delete failed (%v)", err) + } + if results, err := s.Read("foo", store.ReadSuffix()); err != nil { + t.Fatalf("Couldn't read all \"foo\" keys, got %# v (%s)", pretty.Formatter(results), err) + } else { + if len(results) != 0 { + t.Fatalf("Expected 0 items, got %d (%# v)", len(results), pretty.Formatter(results)) + } + } + + // Test Prefix, Suffix and WriteOptions + if err := s.Write(&store.Record{ + Key: "foofoobarbar", + Value: []byte("something"), + }, store.WriteTTL(time.Millisecond*100)); err != nil { + t.Fatal(err) + } + if err := s.Write(&store.Record{ + Key: "foofoo", + Value: []byte("something"), + }, store.WriteExpiry(time.Now().Add(time.Millisecond*100))); err != nil { + t.Fatal(err) + } + if err := s.Write(&store.Record{ + Key: "barbar", + Value: []byte("something"), + // TTL has higher precedence than expiry + }, store.WriteExpiry(time.Now().Add(time.Hour)), store.WriteTTL(time.Millisecond*100)); err != nil { + t.Fatal(err) + } + if results, err := s.Read("foo", store.ReadPrefix(), store.ReadSuffix()); err != nil { + t.Fatal(err) + } else { + if len(results) != 1 { + t.Fatalf("Expected 1 results, got %d: %# v", len(results), pretty.Formatter(results)) + } + } + time.Sleep(time.Second * 6) + if results, err := s.List(); err != nil { + t.Fatalf("List failed: %s", err) + } else { + if len(results) != 0 { + t.Fatal("Expiry options were not effective") + } + } + + s.Init() + for i := 0; i < 10; i++ { + s.Write(&store.Record{ + Key: fmt.Sprintf("a%d", i), + Value: []byte{}, + }) + } + if results, err := s.Read("a", store.ReadLimit(5), store.ReadPrefix()); err != nil { + t.Fatal(err) + } else { + if len(results) != 5 { + t.Fatal("Expected 5 results, got ", len(results)) + } + if results[0].Key != "a0" { + t.Fatalf("Expected a0, got %s", results[0].Key) + } + if results[4].Key != "a4" { + t.Fatalf("Expected a4, got %s", results[4].Key) + } + } + if results, err := s.Read("a", store.ReadLimit(30), store.ReadOffset(5), store.ReadPrefix()); err != nil { + t.Fatal(err) + } else { + if len(results) != 5 { + t.Fatal("Expected 5 results, got ", len(results)) + } + } +} diff --git a/store/memory/memory.go b/store/memory/memory.go index 00ce68b4..d9bdf1b3 100644 --- a/store/memory/memory.go +++ b/store/memory/memory.go @@ -2,154 +2,255 @@ package memory import ( + "sort" "strings" - "sync" "time" "github.com/micro/go-micro/v2/store" + + "github.com/patrickmn/go-cache" + "github.com/pkg/errors" ) +// NewStore returns a memory store +func NewStore(opts ...store.Option) store.Store { + s := &memoryStore{ + options: store.Options{}, + store: cache.New(cache.NoExpiration, 5*time.Minute), + } + for _, o := range opts { + o(&s.options) + } + return s +} + type memoryStore struct { options store.Options - sync.RWMutex - values map[string]*memoryRecord -} - -type memoryRecord struct { - r *store.Record - c time.Time + store *cache.Cache } func (m *memoryStore) Init(opts ...store.Option) error { + m.store.Flush() for _, o := range opts { o(&m.options) } return nil } -func (m *memoryStore) List() ([]*store.Record, error) { - m.RLock() - defer m.RUnlock() - - //nolint:prealloc - var values []*store.Record - - for _, v := range m.values { - // get expiry - d := v.r.Expiry - t := time.Since(v.c) - - if d > time.Duration(0) { - // expired - if t > d { - continue - } - // update expiry - v.r.Expiry -= t - v.c = time.Now() - } - - values = append(values, v.r) - } - - return values, nil -} - -func (m *memoryStore) Read(key string, opts ...store.ReadOption) ([]*store.Record, error) { - m.RLock() - defer m.RUnlock() - - var options store.ReadOptions - - for _, o := range opts { - o(&options) - } - - var vals []*memoryRecord - - if options.Prefix { - for _, v := range m.values { - if !strings.HasPrefix(v.r.Key, key) { - continue - } - vals = append(vals, v) - } - } else if options.Suffix { - for _, v := range m.values { - if !strings.HasSuffix(v.r.Key, key) { - continue - } - vals = append(vals, v) - } - } else { - v, ok := m.values[key] - if !ok { - return nil, store.ErrNotFound - } - vals = []*memoryRecord{v} - } - - //nolint:prealloc - var records []*store.Record - - for _, v := range vals { - // get expiry - d := v.r.Expiry - t := time.Since(v.c) - - // expired - if d > time.Duration(0) { - if t > d { - return nil, store.ErrNotFound - } - // update expiry - v.r.Expiry -= t - v.c = time.Now() - } - - records = append(records, v.r) - } - - return records, nil -} - -func (m *memoryStore) Write(r *store.Record) error { - m.Lock() - defer m.Unlock() - - // set the record - m.values[r.Key] = &memoryRecord{ - r: r, - c: time.Now(), - } - - return nil -} - -func (m *memoryStore) Delete(key string) error { - m.Lock() - defer m.Unlock() - - // delete the value - delete(m.values, key) - - return nil -} - func (m *memoryStore) String() string { return "memory" } -// NewStore returns a new store.Store -func NewStore(opts ...store.Option) store.Store { - var options store.Options +func (m *memoryStore) Read(key string, opts ...store.ReadOption) ([]*store.Record, error) { + readOpts := store.ReadOptions{} for _, o := range opts { - o(&options) + o(&readOpts) } - return &memoryStore{ - options: options, - values: make(map[string]*memoryRecord), + var keys []string + + // Handle Prefix / suffix + if readOpts.Prefix || readOpts.Suffix { + var opts []store.ListOption + if readOpts.Prefix { + opts = append(opts, store.ListPrefix(key)) + } + if readOpts.Suffix { + opts = append(opts, store.ListSuffix(key)) + } + opts = append(opts, store.ListLimit(readOpts.Limit)) + opts = append(opts, store.ListOffset(readOpts.Offset)) + k, err := m.List(opts...) + if err != nil { + return nil, errors.Wrap(err, "Memory: Read couldn't List()") + } + keys = k + } else { + keys = []string{key} } + + var results []*store.Record + for _, k := range keys { + r, err := m.get(k) + if err != nil { + return results, err + } + results = append(results, r) + } + + return results, nil +} + +func (m *memoryStore) get(k string) (*store.Record, error) { + if len(m.options.Suffix) > 0 { + k = k + m.options.Suffix + } + if len(m.options.Prefix) > 0 { + k = m.options.Prefix + "/" + k + } + if len(m.options.Namespace) > 0 { + k = m.options.Namespace + "/" + k + } + var storedRecord *internalRecord + r, found := m.store.Get(k) + if !found { + return nil, store.ErrNotFound + } + storedRecord, ok := r.(*internalRecord) + if !ok { + return nil, errors.New("Retrieved a non *internalRecord from the cache") + } + // Copy the record on the way out + newRecord := &store.Record{} + newRecord.Key = storedRecord.key + newRecord.Value = make([]byte, len(storedRecord.value)) + copy(newRecord.Value, storedRecord.value) + newRecord.Expiry = time.Until(storedRecord.expiresAt) + + return newRecord, nil +} + +func (m *memoryStore) Write(r *store.Record, opts ...store.WriteOption) error { + writeOpts := store.WriteOptions{} + for _, o := range opts { + o(&writeOpts) + } + + if len(opts) > 0 { + // Copy the record before applying options, or the incoming record will be mutated + newRecord := store.Record{} + newRecord.Key = r.Key + newRecord.Value = make([]byte, len(r.Value)) + copy(newRecord.Value, r.Value) + newRecord.Expiry = r.Expiry + + if !writeOpts.Expiry.IsZero() { + newRecord.Expiry = time.Until(writeOpts.Expiry) + } + if writeOpts.TTL != 0 { + newRecord.Expiry = writeOpts.TTL + } + m.set(&newRecord) + } else { + m.set(r) + } + return nil +} + +func (m *memoryStore) set(r *store.Record) { + key := r.Key + if len(m.options.Suffix) > 0 { + key = key + m.options.Suffix + } + if len(m.options.Prefix) > 0 { + key = m.options.Prefix + "/" + key + } + if len(m.options.Namespace) > 0 { + key = m.options.Namespace + "/" + key + } + + // copy the incoming record and then + // convert the expiry in to a hard timestamp + i := &internalRecord{} + i.key = r.Key + i.value = make([]byte, len(r.Value)) + copy(i.value, r.Value) + if r.Expiry != 0 { + i.expiresAt = time.Now().Add(r.Expiry) + } + + m.store.Set(key, i, r.Expiry) +} + +func (m *memoryStore) Delete(key string, opts ...store.DeleteOption) error { + deleteOptions := store.DeleteOptions{} + for _, o := range opts { + o(&deleteOptions) + } + m.delete(key) + return nil +} + +func (m *memoryStore) delete(key string) { + if len(m.options.Suffix) > 0 { + key = key + m.options.Suffix + } + if len(m.options.Prefix) > 0 { + key = m.options.Prefix + "/" + key + } + if len(m.options.Namespace) > 0 { + key = m.options.Namespace + "/" + key + } + m.store.Delete(key) +} + +func (m *memoryStore) Options() store.Options { + return m.options +} + +func (m *memoryStore) List(opts ...store.ListOption) ([]string, error) { + listOptions := store.ListOptions{} + + for _, o := range opts { + o(&listOptions) + } + allKeys := m.list(listOptions.Limit, listOptions.Offset) + + if len(listOptions.Prefix) > 0 { + var prefixKeys []string + for _, k := range allKeys { + if strings.HasPrefix(k, listOptions.Prefix) { + prefixKeys = append(prefixKeys, k) + } + } + allKeys = prefixKeys + } + if len(listOptions.Suffix) > 0 { + var suffixKeys []string + for _, k := range allKeys { + if strings.HasSuffix(k, listOptions.Suffix) { + suffixKeys = append(suffixKeys, k) + } + } + allKeys = suffixKeys + } + + return allKeys, nil +} + +func (m *memoryStore) list(limit, offset uint) []string { + allItems := m.store.Items() + allKeys := make([]string, len(allItems)) + i := 0 + for k := range allItems { + if len(m.options.Suffix) > 0 { + k = strings.TrimSuffix(k, m.options.Suffix) + } + if len(m.options.Namespace) > 0 { + k = strings.TrimPrefix(k, m.options.Namespace+"/") + } + if len(m.options.Prefix) > 0 { + k = strings.TrimPrefix(k, m.options.Prefix+"/") + } + allKeys[i] = k + i++ + } + if limit != 0 || offset != 0 { + sort.Slice(allKeys, func(i, j int) bool { return allKeys[i] < allKeys[j] }) + min := func(i, j uint) uint { + if i < j { + return i + } + return j + } + return allKeys[offset:min(limit, uint(len(allKeys)))] + } + return allKeys +} + +type internalRecord struct { + key string + value []byte + expiresAt time.Time } diff --git a/store/memory/memory_test.go b/store/memory/memory_test.go index 27837e2f..26e04004 100644 --- a/store/memory/memory_test.go +++ b/store/memory/memory_test.go @@ -1,37 +1,265 @@ package memory import ( + "fmt" "testing" "time" + "github.com/kr/pretty" "github.com/micro/go-micro/v2/store" ) -func TestReadRecordExpire(t *testing.T) { - s := NewStore() - - var ( - key = "foo" - expire = 100 * time.Millisecond - ) - rec := &store.Record{ - Key: key, - Value: nil, - Expiry: expire, - } - s.Write(rec) - - rrec, err := s.Read(key) - if err != nil { - t.Fatal(err) - } - if rrec[0].Expiry >= expire { - t.Fatal("expiry of read record is not changed") - } - - time.Sleep(expire) - - if _, err := s.Read(key); err != store.ErrNotFound { - t.Fatal("expire elapsed, but key still accessable") +func TestMemoryReInit(t *testing.T) { + s := NewStore(store.Prefix("aaa")) + s.Init(store.Prefix("")) + if len(s.Options().Prefix) > 0 { + t.Error("Init didn't reinitialise the store") + } +} + +func TestMemoryBasic(t *testing.T) { + s := NewStore() + s.Init() + basictest(s, t) +} + +func TestMemoryPrefix(t *testing.T) { + s := NewStore() + s.Init(store.Prefix("some-prefix")) + basictest(s, t) +} + +func TestMemorySuffix(t *testing.T) { + s := NewStore() + s.Init(store.Suffix("some-suffix")) + basictest(s, t) +} + +func TestMemoryPrefixSuffix(t *testing.T) { + s := NewStore() + s.Init(store.Prefix("some-prefix"), store.Prefix("some-suffix")) + basictest(s, t) +} + +func TestMemoryNamespace(t *testing.T) { + s := NewStore() + s.Init(store.Namespace("some-namespace")) + basictest(s, t) +} + +func TestMemoryNamespacePrefix(t *testing.T) { + s := NewStore() + s.Init(store.Prefix("some-prefix"), store.Namespace("some-namespace")) + basictest(s, t) +} + +func basictest(s store.Store, t *testing.T) { + t.Logf("Testing store %s, with options %# v\n", s.String(), pretty.Formatter(s.Options())) + // Read and Write an expiring Record + if err := s.Write(&store.Record{ + Key: "Hello", + Value: []byte("World"), + Expiry: time.Millisecond * 100, + }); err != nil { + t.Error(err) + } + if r, err := s.Read("Hello"); err != nil { + t.Error(err) + } else { + if len(r) != 1 { + t.Error("Read returned multiple records") + } + if r[0].Key != "Hello" { + t.Errorf("Expected %s, got %s", "Hello", r[0].Key) + } + if string(r[0].Value) != "World" { + t.Errorf("Expected %s, got %s", "World", r[0].Value) + } + } + time.Sleep(time.Millisecond * 200) + if _, err := s.Read("Hello"); err != store.ErrNotFound { + t.Errorf("Expected %# v, got %# v", store.ErrNotFound, err) + } + + // Write 3 records with various expiry and get with prefix + records := []*store.Record{ + &store.Record{ + Key: "foo", + Value: []byte("foofoo"), + }, + &store.Record{ + Key: "foobar", + Value: []byte("foobarfoobar"), + Expiry: time.Millisecond * 100, + }, + &store.Record{ + Key: "foobarbaz", + Value: []byte("foobarbazfoobarbaz"), + Expiry: 2 * time.Millisecond * 100, + }, + } + for _, r := range records { + if err := s.Write(r); err != nil { + t.Errorf("Couldn't write k: %s, v: %# v (%s)", r.Key, pretty.Formatter(r.Value), err) + } + } + if results, err := s.Read("foo", store.ReadPrefix()); err != nil { + t.Errorf("Couldn't read all \"foo\" keys, got %# v (%s)", pretty.Formatter(results), err) + } else { + if len(results) != 3 { + t.Errorf("Expected 3 items, got %d", len(results)) + } + t.Logf("Prefix test: %v\n", pretty.Formatter(results)) + } + time.Sleep(time.Millisecond * 100) + if results, err := s.Read("foo", store.ReadPrefix()); err != nil { + t.Errorf("Couldn't read all \"foo\" keys, got %# v (%s)", pretty.Formatter(results), err) + } else { + if len(results) != 2 { + t.Errorf("Expected 2 items, got %d", len(results)) + } + t.Logf("Prefix test: %v\n", pretty.Formatter(results)) + } + time.Sleep(time.Millisecond * 100) + if results, err := s.Read("foo", store.ReadPrefix()); err != nil { + t.Errorf("Couldn't read all \"foo\" keys, got %# v (%s)", pretty.Formatter(results), err) + } else { + if len(results) != 1 { + t.Errorf("Expected 1 item, got %d", len(results)) + } + t.Logf("Prefix test: %# v\n", pretty.Formatter(results)) + } + if err := s.Delete("foo", func(d *store.DeleteOptions) {}); err != nil { + t.Errorf("Delete failed (%v)", err) + } + if results, err := s.Read("foo", store.ReadPrefix()); err != nil { + t.Errorf("Couldn't read all \"foo\" keys, got %# v (%s)", pretty.Formatter(results), err) + } else { + if len(results) != 0 { + t.Errorf("Expected 0 items, got %d (%# v)", len(results), pretty.Formatter(results)) + } + } + + // Write 3 records with various expiry and get with Suffix + records = []*store.Record{ + &store.Record{ + Key: "foo", + Value: []byte("foofoo"), + }, + &store.Record{ + Key: "barfoo", + Value: []byte("barfoobarfoo"), + Expiry: time.Millisecond * 100, + }, + &store.Record{ + Key: "bazbarfoo", + Value: []byte("bazbarfoobazbarfoo"), + Expiry: 2 * time.Millisecond * 100, + }, + } + for _, r := range records { + if err := s.Write(r); err != nil { + t.Errorf("Couldn't write k: %s, v: %# v (%s)", r.Key, pretty.Formatter(r.Value), err) + } + } + if results, err := s.Read("foo", store.ReadSuffix()); err != nil { + t.Errorf("Couldn't read all \"foo\" keys, got %# v (%s)", pretty.Formatter(results), err) + } else { + if len(results) != 3 { + t.Errorf("Expected 3 items, got %d", len(results)) + } + t.Logf("Prefix test: %v\n", pretty.Formatter(results)) + } + time.Sleep(time.Millisecond * 100) + if results, err := s.Read("foo", store.ReadSuffix()); err != nil { + t.Errorf("Couldn't read all \"foo\" keys, got %# v (%s)", pretty.Formatter(results), err) + } else { + if len(results) != 2 { + t.Errorf("Expected 2 items, got %d", len(results)) + } + t.Logf("Prefix test: %v\n", pretty.Formatter(results)) + } + time.Sleep(time.Millisecond * 100) + if results, err := s.Read("foo", store.ReadSuffix()); err != nil { + t.Errorf("Couldn't read all \"foo\" keys, got %# v (%s)", pretty.Formatter(results), err) + } else { + if len(results) != 1 { + t.Errorf("Expected 1 item, got %d", len(results)) + } + t.Logf("Prefix test: %# v\n", pretty.Formatter(results)) + } + if err := s.Delete("foo"); err != nil { + t.Errorf("Delete failed (%v)", err) + } + if results, err := s.Read("foo", store.ReadSuffix()); err != nil { + t.Errorf("Couldn't read all \"foo\" keys, got %# v (%s)", pretty.Formatter(results), err) + } else { + if len(results) != 0 { + t.Errorf("Expected 0 items, got %d (%# v)", len(results), pretty.Formatter(results)) + } + } + + // Test Prefix, Suffix and WriteOptions + if err := s.Write(&store.Record{ + Key: "foofoobarbar", + Value: []byte("something"), + }, store.WriteTTL(time.Millisecond*100)); err != nil { + t.Error(err) + } + if err := s.Write(&store.Record{ + Key: "foofoo", + Value: []byte("something"), + }, store.WriteExpiry(time.Now().Add(time.Millisecond*100))); err != nil { + t.Error(err) + } + if err := s.Write(&store.Record{ + Key: "barbar", + Value: []byte("something"), + // TTL has higher precedence than expiry + }, store.WriteExpiry(time.Now().Add(time.Hour)), store.WriteTTL(time.Millisecond*100)); err != nil { + t.Error(err) + } + if results, err := s.Read("foo", store.ReadPrefix(), store.ReadSuffix()); err != nil { + t.Error(err) + } else { + if len(results) != 1 { + t.Errorf("Expected 1 results, got %d: %# v", len(results), pretty.Formatter(results)) + } + } + time.Sleep(time.Millisecond * 100) + if results, err := s.List(); err != nil { + t.Errorf("List failed: %s", err) + } else { + if len(results) != 0 { + t.Error("Expiry options were not effective") + } + } + + s.Init() + for i := 0; i < 10; i++ { + s.Write(&store.Record{ + Key: fmt.Sprintf("a%d", i), + Value: []byte{}, + }) + } + if results, err := s.Read("a", store.ReadLimit(5), store.ReadPrefix()); err != nil { + t.Error(err) + } else { + if len(results) != 5 { + t.Error("Expected 5 results, got ", len(results)) + } + if results[0].Key != "a0" { + t.Errorf("Expected a0, got %s", results[0].Key) + } + if results[4].Key != "a4" { + t.Errorf("Expected a4, got %s", results[4].Key) + } + } + if results, err := s.Read("a", store.ReadLimit(30), store.ReadOffset(5), store.ReadPrefix()); err != nil { + t.Error(err) + } else { + if len(results) != 5 { + t.Error("Expected 5 results, got ", len(results)) + } } } diff --git a/store/noop.go b/store/noop.go new file mode 100644 index 00000000..0c6ebb4b --- /dev/null +++ b/store/noop.go @@ -0,0 +1,31 @@ +package store + +type noopStore struct{} + +func (n *noopStore) Init(opts ...Option) error { + return nil +} + +func (n *noopStore) Options() Options { + return Options{} +} + +func (n *noopStore) String() string { + return "noop" +} + +func (n *noopStore) Read(key string, opts ...ReadOption) ([]*Record, error) { + return []*Record{}, nil +} + +func (n *noopStore) Write(r *Record, opts ...WriteOption) error { + return nil +} + +func (n *noopStore) Delete(key string, opts ...DeleteOption) error { + return nil +} + +func (n *noopStore) List(opts ...ListOption) ([]string, error) { + return []string{}, nil +} diff --git a/store/options.go b/store/options.go index f1ed4f52..98b9915a 100644 --- a/store/options.go +++ b/store/options.go @@ -2,55 +2,181 @@ package store import ( "context" + "time" ) +// Options contains configuration for the Store type Options struct { - // nodes to connect to + // Nodes contains the addresses or other connection information of the backing storage. + // For example, an etcd implementation would contain the nodes of the cluster. + // A SQL implementation could contain one or more connection strings. Nodes []string - // Namespace of the store + // Namespace allows multiple isolated stores to be kept in one backend, if supported. + // For example multiple tables in a SQL store. Namespace string - // Prefix of the keys used + // Prefix sets a global prefix on all keys Prefix string - // Suffix of the keys used + // Suffix sets a global suffix on all keys Suffix string - // Alternative options + // Context should contain all implementation specific options, using context.WithValue. Context context.Context } +// Option sets values in Options type Option func(o *Options) -// Nodes is a list of nodes used to back the store +// Nodes contains the addresses or other connection information of the backing storage. +// For example, an etcd implementation would contain the nodes of the cluster. +// A SQL implementation could contain one or more connection strings. func Nodes(a ...string) Option { return func(o *Options) { o.Nodes = a } } -// Prefix sets a prefix to any key ids used -func Prefix(p string) Option { - return func(o *Options) { - o.Prefix = p - } -} - -// Namespace offers a way to have multiple isolated -// stores in the same backend, if supported. +// Namespace allows multiple isolated stores to be kept in one backend, if supported. +// For example multiple tables in a SQL store. func Namespace(ns string) Option { return func(o *Options) { o.Namespace = ns } } -// ReadPrefix uses the key as a prefix -func ReadPrefix() ReadOption { - return func(o *ReadOptions) { - o.Prefix = true +// Prefix sets a global prefix on all keys +func Prefix(p string) Option { + return func(o *Options) { + o.Prefix = p } } -// ReadSuffix uses the key as a suffix -func ReadSuffix() ReadOption { - return func(o *ReadOptions) { - o.Suffix = true +// Suffix sets a global suffix on all keys +func Suffix(s string) Option { + return func(o *Options) { + o.Suffix = s + } +} + +// WithContext sets the stores context, for any extra configuration +func WithContext(c context.Context) Option { + return func(o *Options) { + o.Context = c + } +} + +// ReadOptions configures an individual Read operation +type ReadOptions struct { + // Prefix returns all records that are prefixed with key + Prefix bool + // Suffix returns all records that have the suffix key + Suffix bool + // Limit limits the number of returned records + Limit uint + // Offset when combined with Limit supports pagination + Offset uint +} + +// ReadOption sets values in ReadOptions +type ReadOption func(r *ReadOptions) + +// ReadPrefix returns all records that are prefixed with key +func ReadPrefix() ReadOption { + return func(r *ReadOptions) { + r.Prefix = true + } +} + +// ReadSuffix returns all records that have the suffix key +func ReadSuffix() ReadOption { + return func(r *ReadOptions) { + r.Suffix = true + } +} + +// ReadLimit limits the number of responses to l +func ReadLimit(l uint) ReadOption { + return func(r *ReadOptions) { + r.Limit = l + } +} + +// ReadOffset starts returning responses from o. Use in conjunction with Limit for pagination +func ReadOffset(o uint) ReadOption { + return func(r *ReadOptions) { + r.Offset = o + } +} + +// WriteOptions configures an individual Write operation +// If Expiry and TTL are set TTL takes precedence +type WriteOptions struct { + // Expiry is the time the record expires + Expiry time.Time + // TTL is the time until the record expires + TTL time.Duration +} + +// WriteOption sets values in WriteOptions +type WriteOption func(w *WriteOptions) + +// WriteExpiry is the time the record expires +func WriteExpiry(t time.Time) WriteOption { + return func(w *WriteOptions) { + w.Expiry = t + } +} + +// WriteTTL is the time the record expires +func WriteTTL(d time.Duration) WriteOption { + return func(w *WriteOptions) { + w.TTL = d + } +} + +// DeleteOptions configures an individual Delete operation +type DeleteOptions struct{} + +// DeleteOption sets values in DeleteOptions +type DeleteOption func(d *DeleteOptions) + +// ListOptions configures an individual List operation +type ListOptions struct { + // Prefix returns all keys that are prefixed with key + Prefix string + // Suffix returns all keys that end with key + Suffix string + // Limit limits the number of returned keys + Limit uint + // Offset when combined with Limit supports pagination + Offset uint +} + +// ListOption sets values in ListOptions +type ListOption func(l *ListOptions) + +// ListPrefix returns all keys that are prefixed with key +func ListPrefix(p string) ListOption { + return func(l *ListOptions) { + l.Prefix = p + } +} + +// ListSuffix returns all keys that end with key +func ListSuffix(s string) ListOption { + return func(l *ListOptions) { + l.Suffix = s + } +} + +// ListLimit limits the number of returned keys to l +func ListLimit(l uint) ListOption { + return func(lo *ListOptions) { + lo.Limit = l + } +} + +// ListOffset starts returning responses from o. Use in conjunction with Limit for pagination. +func ListOffset(o uint) ListOption { + return func(l *ListOptions) { + l.Offset = o } } diff --git a/store/scope/scope.go b/store/scope/scope.go new file mode 100644 index 00000000..c78e1e1a --- /dev/null +++ b/store/scope/scope.go @@ -0,0 +1,51 @@ +package scope + +import ( + "fmt" + + "github.com/micro/go-micro/v2/store" +) + +// Scope extends the store, applying a prefix to each request +type Scope struct { + store.Store + prefix string +} + +// NewScope returns an initialised scope +func NewScope(s store.Store, prefix string) Scope { + return Scope{Store: s, prefix: prefix} +} + +func (s *Scope) Options() store.Options { + o := s.Store.Options() + o.Prefix = s.prefix + return o +} + +func (s *Scope) Read(key string, opts ...store.ReadOption) ([]*store.Record, error) { + key = fmt.Sprintf("%v/%v", s.prefix, key) + return s.Store.Read(key, opts...) +} + +func (s *Scope) Write(r *store.Record, opts ...store.WriteOption) error { + r.Key = fmt.Sprintf("%v/%v", s.prefix, r.Key) + return s.Store.Write(r, opts...) +} + +func (s *Scope) Delete(key string, opts ...store.DeleteOption) error { + key = fmt.Sprintf("%v/%v", s.prefix, key) + return s.Store.Delete(key, opts...) +} + +func (s *Scope) List(opts ...store.ListOption) ([]string, error) { + var lops store.ListOptions + for _, o := range opts { + o(&lops) + } + + key := fmt.Sprintf("%v/%v", s.prefix, lops.Prefix) + opts = append(opts, store.ListPrefix(key)) + + return s.Store.List(opts...) +} diff --git a/store/service/proto/store.pb.go b/store/service/proto/store.pb.go index 6a05463a..b7cd9979 100644 --- a/store/service/proto/store.pb.go +++ b/store/service/proto/store.pb.go @@ -1,5 +1,5 @@ // Code generated by protoc-gen-go. DO NOT EDIT. -// source: micro/go-micro/store/service/proto/store.proto +// source: store.proto package go_micro_store @@ -25,7 +25,7 @@ type Record struct { Key string `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"` // value in the record Value []byte `protobuf:"bytes,2,opt,name=value,proto3" json:"value,omitempty"` - // timestamp in unix seconds + // time.Duration (signed int64 nanoseconds) Expiry int64 `protobuf:"varint,3,opt,name=expiry,proto3" json:"expiry,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` @@ -36,7 +36,7 @@ func (m *Record) Reset() { *m = Record{} } func (m *Record) String() string { return proto.CompactTextString(m) } func (*Record) ProtoMessage() {} func (*Record) Descriptor() ([]byte, []int) { - return fileDescriptor_f84ccc98e143ed3e, []int{0} + return fileDescriptor_98bbca36ef968dfc, []int{0} } func (m *Record) XXX_Unmarshal(b []byte) error { @@ -80,6 +80,9 @@ func (m *Record) GetExpiry() int64 { type ReadOptions struct { Prefix bool `protobuf:"varint,1,opt,name=prefix,proto3" json:"prefix,omitempty"` + Suffix bool `protobuf:"varint,2,opt,name=suffix,proto3" json:"suffix,omitempty"` + Limit uint64 `protobuf:"varint,3,opt,name=limit,proto3" json:"limit,omitempty"` + Offset uint64 `protobuf:"varint,4,opt,name=offset,proto3" json:"offset,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` @@ -89,7 +92,7 @@ func (m *ReadOptions) Reset() { *m = ReadOptions{} } func (m *ReadOptions) String() string { return proto.CompactTextString(m) } func (*ReadOptions) ProtoMessage() {} func (*ReadOptions) Descriptor() ([]byte, []int) { - return fileDescriptor_f84ccc98e143ed3e, []int{1} + return fileDescriptor_98bbca36ef968dfc, []int{1} } func (m *ReadOptions) XXX_Unmarshal(b []byte) error { @@ -117,6 +120,27 @@ func (m *ReadOptions) GetPrefix() bool { return false } +func (m *ReadOptions) GetSuffix() bool { + if m != nil { + return m.Suffix + } + return false +} + +func (m *ReadOptions) GetLimit() uint64 { + if m != nil { + return m.Limit + } + return 0 +} + +func (m *ReadOptions) GetOffset() uint64 { + if m != nil { + return m.Offset + } + return 0 +} + type ReadRequest struct { Key string `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"` Options *ReadOptions `protobuf:"bytes,2,opt,name=options,proto3" json:"options,omitempty"` @@ -129,7 +153,7 @@ func (m *ReadRequest) Reset() { *m = ReadRequest{} } func (m *ReadRequest) String() string { return proto.CompactTextString(m) } func (*ReadRequest) ProtoMessage() {} func (*ReadRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_f84ccc98e143ed3e, []int{2} + return fileDescriptor_98bbca36ef968dfc, []int{2} } func (m *ReadRequest) XXX_Unmarshal(b []byte) error { @@ -175,7 +199,7 @@ func (m *ReadResponse) Reset() { *m = ReadResponse{} } func (m *ReadResponse) String() string { return proto.CompactTextString(m) } func (*ReadResponse) ProtoMessage() {} func (*ReadResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_f84ccc98e143ed3e, []int{3} + return fileDescriptor_98bbca36ef968dfc, []int{3} } func (m *ReadResponse) XXX_Unmarshal(b []byte) error { @@ -203,18 +227,68 @@ func (m *ReadResponse) GetRecords() []*Record { return nil } -type WriteRequest struct { - Record *Record `protobuf:"bytes,1,opt,name=record,proto3" json:"record,omitempty"` +type WriteOptions struct { + // time.Time + Expiry int64 `protobuf:"varint,1,opt,name=expiry,proto3" json:"expiry,omitempty"` + // time.Duration + Ttl int64 `protobuf:"varint,2,opt,name=ttl,proto3" json:"ttl,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } +func (m *WriteOptions) Reset() { *m = WriteOptions{} } +func (m *WriteOptions) String() string { return proto.CompactTextString(m) } +func (*WriteOptions) ProtoMessage() {} +func (*WriteOptions) Descriptor() ([]byte, []int) { + return fileDescriptor_98bbca36ef968dfc, []int{4} +} + +func (m *WriteOptions) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_WriteOptions.Unmarshal(m, b) +} +func (m *WriteOptions) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_WriteOptions.Marshal(b, m, deterministic) +} +func (m *WriteOptions) XXX_Merge(src proto.Message) { + xxx_messageInfo_WriteOptions.Merge(m, src) +} +func (m *WriteOptions) XXX_Size() int { + return xxx_messageInfo_WriteOptions.Size(m) +} +func (m *WriteOptions) XXX_DiscardUnknown() { + xxx_messageInfo_WriteOptions.DiscardUnknown(m) +} + +var xxx_messageInfo_WriteOptions proto.InternalMessageInfo + +func (m *WriteOptions) GetExpiry() int64 { + if m != nil { + return m.Expiry + } + return 0 +} + +func (m *WriteOptions) GetTtl() int64 { + if m != nil { + return m.Ttl + } + return 0 +} + +type WriteRequest struct { + Record *Record `protobuf:"bytes,1,opt,name=record,proto3" json:"record,omitempty"` + Options *WriteOptions `protobuf:"bytes,2,opt,name=options,proto3" json:"options,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + func (m *WriteRequest) Reset() { *m = WriteRequest{} } func (m *WriteRequest) String() string { return proto.CompactTextString(m) } func (*WriteRequest) ProtoMessage() {} func (*WriteRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_f84ccc98e143ed3e, []int{4} + return fileDescriptor_98bbca36ef968dfc, []int{5} } func (m *WriteRequest) XXX_Unmarshal(b []byte) error { @@ -242,6 +316,13 @@ func (m *WriteRequest) GetRecord() *Record { return nil } +func (m *WriteRequest) GetOptions() *WriteOptions { + if m != nil { + return m.Options + } + return nil +} + type WriteResponse struct { XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` @@ -252,7 +333,7 @@ func (m *WriteResponse) Reset() { *m = WriteResponse{} } func (m *WriteResponse) String() string { return proto.CompactTextString(m) } func (*WriteResponse) ProtoMessage() {} func (*WriteResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_f84ccc98e143ed3e, []int{5} + return fileDescriptor_98bbca36ef968dfc, []int{6} } func (m *WriteResponse) XXX_Unmarshal(b []byte) error { @@ -273,18 +354,50 @@ func (m *WriteResponse) XXX_DiscardUnknown() { var xxx_messageInfo_WriteResponse proto.InternalMessageInfo -type DeleteRequest struct { - Key string `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"` +type DeleteOptions struct { XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } +func (m *DeleteOptions) Reset() { *m = DeleteOptions{} } +func (m *DeleteOptions) String() string { return proto.CompactTextString(m) } +func (*DeleteOptions) ProtoMessage() {} +func (*DeleteOptions) Descriptor() ([]byte, []int) { + return fileDescriptor_98bbca36ef968dfc, []int{7} +} + +func (m *DeleteOptions) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_DeleteOptions.Unmarshal(m, b) +} +func (m *DeleteOptions) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_DeleteOptions.Marshal(b, m, deterministic) +} +func (m *DeleteOptions) XXX_Merge(src proto.Message) { + xxx_messageInfo_DeleteOptions.Merge(m, src) +} +func (m *DeleteOptions) XXX_Size() int { + return xxx_messageInfo_DeleteOptions.Size(m) +} +func (m *DeleteOptions) XXX_DiscardUnknown() { + xxx_messageInfo_DeleteOptions.DiscardUnknown(m) +} + +var xxx_messageInfo_DeleteOptions proto.InternalMessageInfo + +type DeleteRequest struct { + Key string `protobuf:"bytes,1,opt,name=key,proto3" json:"key,omitempty"` + Options *DeleteOptions `protobuf:"bytes,2,opt,name=options,proto3" json:"options,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + func (m *DeleteRequest) Reset() { *m = DeleteRequest{} } func (m *DeleteRequest) String() string { return proto.CompactTextString(m) } func (*DeleteRequest) ProtoMessage() {} func (*DeleteRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_f84ccc98e143ed3e, []int{6} + return fileDescriptor_98bbca36ef968dfc, []int{8} } func (m *DeleteRequest) XXX_Unmarshal(b []byte) error { @@ -312,6 +425,13 @@ func (m *DeleteRequest) GetKey() string { return "" } +func (m *DeleteRequest) GetOptions() *DeleteOptions { + if m != nil { + return m.Options + } + return nil +} + type DeleteResponse struct { XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` @@ -322,7 +442,7 @@ func (m *DeleteResponse) Reset() { *m = DeleteResponse{} } func (m *DeleteResponse) String() string { return proto.CompactTextString(m) } func (*DeleteResponse) ProtoMessage() {} func (*DeleteResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_f84ccc98e143ed3e, []int{7} + return fileDescriptor_98bbca36ef968dfc, []int{9} } func (m *DeleteResponse) XXX_Unmarshal(b []byte) error { @@ -343,17 +463,81 @@ func (m *DeleteResponse) XXX_DiscardUnknown() { var xxx_messageInfo_DeleteResponse proto.InternalMessageInfo -type ListRequest struct { +type ListOptions struct { + Prefix string `protobuf:"bytes,1,opt,name=prefix,proto3" json:"prefix,omitempty"` + Suffix string `protobuf:"bytes,2,opt,name=suffix,proto3" json:"suffix,omitempty"` + Limit uint64 `protobuf:"varint,3,opt,name=limit,proto3" json:"limit,omitempty"` + Offset uint64 `protobuf:"varint,4,opt,name=offset,proto3" json:"offset,omitempty"` XXX_NoUnkeyedLiteral struct{} `json:"-"` XXX_unrecognized []byte `json:"-"` XXX_sizecache int32 `json:"-"` } +func (m *ListOptions) Reset() { *m = ListOptions{} } +func (m *ListOptions) String() string { return proto.CompactTextString(m) } +func (*ListOptions) ProtoMessage() {} +func (*ListOptions) Descriptor() ([]byte, []int) { + return fileDescriptor_98bbca36ef968dfc, []int{10} +} + +func (m *ListOptions) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_ListOptions.Unmarshal(m, b) +} +func (m *ListOptions) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_ListOptions.Marshal(b, m, deterministic) +} +func (m *ListOptions) XXX_Merge(src proto.Message) { + xxx_messageInfo_ListOptions.Merge(m, src) +} +func (m *ListOptions) XXX_Size() int { + return xxx_messageInfo_ListOptions.Size(m) +} +func (m *ListOptions) XXX_DiscardUnknown() { + xxx_messageInfo_ListOptions.DiscardUnknown(m) +} + +var xxx_messageInfo_ListOptions proto.InternalMessageInfo + +func (m *ListOptions) GetPrefix() string { + if m != nil { + return m.Prefix + } + return "" +} + +func (m *ListOptions) GetSuffix() string { + if m != nil { + return m.Suffix + } + return "" +} + +func (m *ListOptions) GetLimit() uint64 { + if m != nil { + return m.Limit + } + return 0 +} + +func (m *ListOptions) GetOffset() uint64 { + if m != nil { + return m.Offset + } + return 0 +} + +type ListRequest struct { + Options *ListOptions `protobuf:"bytes,1,opt,name=options,proto3" json:"options,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + func (m *ListRequest) Reset() { *m = ListRequest{} } func (m *ListRequest) String() string { return proto.CompactTextString(m) } func (*ListRequest) ProtoMessage() {} func (*ListRequest) Descriptor() ([]byte, []int) { - return fileDescriptor_f84ccc98e143ed3e, []int{8} + return fileDescriptor_98bbca36ef968dfc, []int{11} } func (m *ListRequest) XXX_Unmarshal(b []byte) error { @@ -374,18 +558,25 @@ func (m *ListRequest) XXX_DiscardUnknown() { var xxx_messageInfo_ListRequest proto.InternalMessageInfo +func (m *ListRequest) GetOptions() *ListOptions { + if m != nil { + return m.Options + } + return nil +} + type ListResponse struct { - Records []*Record `protobuf:"bytes,1,rep,name=records,proto3" json:"records,omitempty"` - XXX_NoUnkeyedLiteral struct{} `json:"-"` - XXX_unrecognized []byte `json:"-"` - XXX_sizecache int32 `json:"-"` + Keys []string `protobuf:"bytes,2,rep,name=keys,proto3" json:"keys,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` } func (m *ListResponse) Reset() { *m = ListResponse{} } func (m *ListResponse) String() string { return proto.CompactTextString(m) } func (*ListResponse) ProtoMessage() {} func (*ListResponse) Descriptor() ([]byte, []int) { - return fileDescriptor_f84ccc98e143ed3e, []int{9} + return fileDescriptor_98bbca36ef968dfc, []int{12} } func (m *ListResponse) XXX_Unmarshal(b []byte) error { @@ -406,9 +597,9 @@ func (m *ListResponse) XXX_DiscardUnknown() { var xxx_messageInfo_ListResponse proto.InternalMessageInfo -func (m *ListResponse) GetRecords() []*Record { +func (m *ListResponse) GetKeys() []string { if m != nil { - return m.Records + return m.Keys } return nil } @@ -418,41 +609,48 @@ func init() { proto.RegisterType((*ReadOptions)(nil), "go.micro.store.ReadOptions") proto.RegisterType((*ReadRequest)(nil), "go.micro.store.ReadRequest") proto.RegisterType((*ReadResponse)(nil), "go.micro.store.ReadResponse") + proto.RegisterType((*WriteOptions)(nil), "go.micro.store.WriteOptions") proto.RegisterType((*WriteRequest)(nil), "go.micro.store.WriteRequest") proto.RegisterType((*WriteResponse)(nil), "go.micro.store.WriteResponse") + proto.RegisterType((*DeleteOptions)(nil), "go.micro.store.DeleteOptions") proto.RegisterType((*DeleteRequest)(nil), "go.micro.store.DeleteRequest") proto.RegisterType((*DeleteResponse)(nil), "go.micro.store.DeleteResponse") + proto.RegisterType((*ListOptions)(nil), "go.micro.store.ListOptions") proto.RegisterType((*ListRequest)(nil), "go.micro.store.ListRequest") proto.RegisterType((*ListResponse)(nil), "go.micro.store.ListResponse") } -func init() { - proto.RegisterFile("micro/go-micro/store/service/proto/store.proto", fileDescriptor_f84ccc98e143ed3e) -} +func init() { proto.RegisterFile("store.proto", fileDescriptor_98bbca36ef968dfc) } -var fileDescriptor_f84ccc98e143ed3e = []byte{ - // 364 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x53, 0x4d, 0x4b, 0xc3, 0x40, - 0x10, 0x6d, 0x9a, 0x36, 0xd5, 0x49, 0x5b, 0xcb, 0x22, 0x25, 0xd4, 0x0f, 0xe2, 0x82, 0x90, 0x8b, - 0x69, 0xa9, 0x78, 0x15, 0xc1, 0x0f, 0x14, 0x04, 0x61, 0x05, 0x3d, 0xd7, 0x76, 0x2c, 0xc1, 0xda, - 0x8d, 0xbb, 0x69, 0x69, 0xff, 0x90, 0xbf, 0x53, 0xb2, 0xbb, 0xd1, 0x94, 0x34, 0x17, 0x6f, 0x33, - 0xfb, 0xde, 0xbc, 0x99, 0x79, 0xc3, 0x42, 0xf8, 0x19, 0x8d, 0x05, 0xef, 0x4f, 0xf9, 0x99, 0x0e, - 0x64, 0xc2, 0x05, 0xf6, 0x25, 0x8a, 0x65, 0x34, 0xc6, 0x7e, 0x2c, 0x78, 0x62, 0xde, 0x42, 0x15, - 0x93, 0xf6, 0x94, 0xeb, 0x92, 0x50, 0xbd, 0xd2, 0x7b, 0x70, 0x18, 0x8e, 0xb9, 0x98, 0x90, 0x0e, - 0xd8, 0x1f, 0xb8, 0xf6, 0x2c, 0xdf, 0x0a, 0x76, 0x59, 0x1a, 0x92, 0x7d, 0xa8, 0x2f, 0x47, 0xb3, - 0x05, 0x7a, 0x55, 0xdf, 0x0a, 0x9a, 0x4c, 0x27, 0xa4, 0x0b, 0x0e, 0xae, 0xe2, 0x48, 0xac, 0x3d, - 0xdb, 0xb7, 0x02, 0x9b, 0x99, 0x8c, 0x9e, 0x82, 0xcb, 0x70, 0x34, 0x79, 0x8a, 0x93, 0x88, 0xcf, - 0x65, 0x4a, 0x8b, 0x05, 0xbe, 0x47, 0x2b, 0xa5, 0xb8, 0xc3, 0x4c, 0x46, 0x5f, 0x34, 0x8d, 0xe1, - 0xd7, 0x02, 0x65, 0xb2, 0xa5, 0xeb, 0x05, 0x34, 0xb8, 0xd6, 0x50, 0x7d, 0xdd, 0xe1, 0x41, 0xb8, - 0x39, 0x73, 0x98, 0x6b, 0xc3, 0x32, 0x2e, 0xbd, 0x82, 0xa6, 0xd6, 0x95, 0x31, 0x9f, 0x4b, 0x24, - 0x03, 0x68, 0x08, 0xb5, 0x98, 0xf4, 0x2c, 0xdf, 0x0e, 0xdc, 0x61, 0xb7, 0x28, 0x93, 0xc2, 0x2c, - 0xa3, 0xd1, 0x4b, 0x68, 0xbe, 0x8a, 0x28, 0xc1, 0x6c, 0xb4, 0x10, 0x1c, 0x0d, 0xa9, 0xe9, 0xca, - 0x05, 0x0c, 0x8b, 0xee, 0x41, 0xcb, 0xd4, 0xeb, 0x11, 0xe8, 0x09, 0xb4, 0x6e, 0x70, 0x86, 0x7f, - 0x8a, 0x85, 0x65, 0x69, 0x07, 0xda, 0x19, 0xc5, 0x14, 0xb5, 0xc0, 0x7d, 0x8c, 0x64, 0x62, 0x4a, - 0xd2, 0xb5, 0x74, 0xfa, 0xdf, 0xb5, 0x86, 0xdf, 0x55, 0xa8, 0x3f, 0xa7, 0x08, 0xb9, 0x85, 0x5a, - 0xaa, 0x45, 0x0a, 0x86, 0xe6, 0x1a, 0xf6, 0x0e, 0xb7, 0x83, 0x66, 0xba, 0xca, 0xc0, 0x22, 0xd7, - 0x50, 0x4b, 0x9d, 0x26, 0x5b, 0xef, 0x52, 0x2a, 0x93, 0x3f, 0x0e, 0xad, 0x90, 0x3b, 0xa8, 0x2b, - 0xb3, 0x48, 0x81, 0x98, 0xbf, 0x41, 0xef, 0xa8, 0x04, 0xfd, 0xd5, 0x79, 0x00, 0x47, 0x1b, 0x48, - 0x0a, 0xd4, 0x0d, 0xef, 0x7b, 0xc7, 0x65, 0x70, 0x26, 0xf5, 0xe6, 0xa8, 0x1f, 0x72, 0xfe, 0x13, - 0x00, 0x00, 0xff, 0xff, 0xf9, 0x20, 0x54, 0x71, 0x53, 0x03, 0x00, 0x00, +var fileDescriptor_98bbca36ef968dfc = []byte{ + // 463 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x9c, 0x54, 0x5d, 0x6b, 0xd4, 0x40, + 0x14, 0xed, 0x6c, 0xb2, 0x69, 0xf7, 0x26, 0xd6, 0x65, 0x90, 0x12, 0xb4, 0x95, 0x30, 0x4f, 0x79, + 0x0a, 0x65, 0xc5, 0x8f, 0x47, 0xc1, 0x2a, 0x2a, 0x82, 0x30, 0x82, 0x82, 0x6f, 0xb5, 0x9d, 0xc8, + 0x90, 0xb4, 0x13, 0x67, 0x66, 0x4b, 0xf7, 0x0f, 0xf9, 0x3b, 0x65, 0xbe, 0x76, 0xb3, 0x31, 0x79, + 0xe9, 0x5b, 0xee, 0x9d, 0x3b, 0xe7, 0xdc, 0x73, 0xe6, 0x10, 0x48, 0x95, 0x16, 0x92, 0x55, 0x9d, + 0x14, 0x5a, 0xe0, 0xe3, 0xdf, 0xa2, 0xba, 0xe1, 0x57, 0x52, 0x54, 0xb6, 0x4b, 0x3e, 0x42, 0x42, + 0xd9, 0x95, 0x90, 0xd7, 0x78, 0x09, 0x51, 0xc3, 0x36, 0x39, 0x2a, 0x50, 0xb9, 0xa0, 0xe6, 0x13, + 0x3f, 0x81, 0xf9, 0xdd, 0x65, 0xbb, 0x66, 0xf9, 0xac, 0x40, 0x65, 0x46, 0x5d, 0x81, 0x4f, 0x20, + 0x61, 0xf7, 0x1d, 0x97, 0x9b, 0x3c, 0x2a, 0x50, 0x19, 0x51, 0x5f, 0x91, 0x06, 0x52, 0xca, 0x2e, + 0xaf, 0xbf, 0x76, 0x9a, 0x8b, 0x5b, 0x65, 0xc6, 0x3a, 0xc9, 0x6a, 0x7e, 0x6f, 0x11, 0x8f, 0xa8, + 0xaf, 0x4c, 0x5f, 0xad, 0x6b, 0xd3, 0x9f, 0xb9, 0xbe, 0xab, 0x0c, 0x59, 0xcb, 0x6f, 0xb8, 0xb6, + 0xa8, 0x31, 0x75, 0x85, 0x99, 0x16, 0x75, 0xad, 0x98, 0xce, 0x63, 0xdb, 0xf6, 0x15, 0xf9, 0xee, + 0xc8, 0x28, 0xfb, 0xb3, 0x66, 0x4a, 0x8f, 0xec, 0xfe, 0x12, 0x0e, 0x85, 0xdb, 0xc4, 0xf2, 0xa4, + 0xab, 0x67, 0xd5, 0xbe, 0xf2, 0xaa, 0xb7, 0x2c, 0x0d, 0xb3, 0xe4, 0x2d, 0x64, 0x0e, 0x57, 0x75, + 0xe2, 0x56, 0x31, 0x7c, 0x0e, 0x87, 0xd2, 0xda, 0xa3, 0x72, 0x54, 0x44, 0x65, 0xba, 0x3a, 0xf9, + 0x1f, 0xc6, 0x1c, 0xd3, 0x30, 0x46, 0xde, 0x40, 0xf6, 0x43, 0x72, 0xcd, 0x7a, 0x3e, 0x78, 0xbb, + 0x50, 0xdf, 0x2e, 0xb3, 0xb2, 0xd6, 0xad, 0x5d, 0x2e, 0xa2, 0xe6, 0x93, 0xdc, 0xf9, 0x9b, 0x41, + 0x54, 0x05, 0x89, 0x03, 0xb5, 0x37, 0xa7, 0xa9, 0xfd, 0x14, 0x7e, 0x35, 0x94, 0x7c, 0x3a, 0xbc, + 0xd0, 0x5f, 0x6c, 0xa7, 0xf9, 0x31, 0x3c, 0xf2, 0xbc, 0x4e, 0xb4, 0x69, 0x5c, 0xb0, 0x96, 0x6d, + 0x47, 0xc9, 0xcf, 0xd0, 0x98, 0xf6, 0xfb, 0xf5, 0x90, 0xfc, 0x6c, 0x48, 0xbe, 0x07, 0xb9, 0x63, + 0x5f, 0xc2, 0x71, 0xc0, 0xf6, 0xf4, 0x0d, 0xa4, 0x5f, 0xb8, 0xd2, 0xe3, 0x41, 0x5a, 0x4c, 0x04, + 0x69, 0xf1, 0xc0, 0x20, 0x5d, 0x38, 0xb2, 0x20, 0xac, 0x17, 0x1b, 0x34, 0x1e, 0x9b, 0xde, 0x6a, + 0x3b, 0x11, 0x25, 0x64, 0x0e, 0xc5, 0xc7, 0x06, 0x43, 0xdc, 0xb0, 0x8d, 0xb1, 0x22, 0x2a, 0x17, + 0xd4, 0x7e, 0x7f, 0x8e, 0x8f, 0xd0, 0x72, 0xb6, 0xfa, 0x3b, 0x83, 0xf9, 0x37, 0x03, 0x84, 0xdf, + 0x41, 0x6c, 0xa2, 0x86, 0x47, 0x83, 0xe9, 0xf7, 0x79, 0x7a, 0x3a, 0x7e, 0xe8, 0x9d, 0x3a, 0xc0, + 0x1f, 0x60, 0x6e, 0xdf, 0x0e, 0x8f, 0xbf, 0x75, 0x80, 0x39, 0x9b, 0x38, 0xdd, 0xe2, 0x7c, 0x82, + 0xc4, 0xbd, 0x02, 0x9e, 0x78, 0xb7, 0x80, 0xf4, 0x7c, 0xea, 0x78, 0x0b, 0xf5, 0x1e, 0x62, 0xe3, + 0x05, 0x1e, 0x75, 0x6e, 0x52, 0x57, 0xdf, 0x3e, 0x72, 0x70, 0x8e, 0x7e, 0x25, 0xf6, 0x7f, 0xf5, + 0xe2, 0x5f, 0x00, 0x00, 0x00, 0xff, 0xff, 0xa5, 0x11, 0xc3, 0x50, 0xbe, 0x04, 0x00, 0x00, } diff --git a/store/service/proto/store.pb.micro.go b/store/service/proto/store.pb.micro.go index 448ea6bc..321449b5 100644 --- a/store/service/proto/store.pb.micro.go +++ b/store/service/proto/store.pb.micro.go @@ -1,5 +1,5 @@ // Code generated by protoc-gen-micro. DO NOT EDIT. -// source: micro/go-micro/store/service/proto/store.proto +// source: store.proto package go_micro_store @@ -34,10 +34,10 @@ var _ server.Option // Client API for Store service type StoreService interface { - List(ctx context.Context, in *ListRequest, opts ...client.CallOption) (Store_ListService, error) Read(ctx context.Context, in *ReadRequest, opts ...client.CallOption) (*ReadResponse, error) Write(ctx context.Context, in *WriteRequest, opts ...client.CallOption) (*WriteResponse, error) Delete(ctx context.Context, in *DeleteRequest, opts ...client.CallOption) (*DeleteResponse, error) + List(ctx context.Context, in *ListRequest, opts ...client.CallOption) (Store_ListService, error) } type storeService struct { @@ -46,62 +46,12 @@ type storeService struct { } func NewStoreService(name string, c client.Client) StoreService { - if c == nil { - c = client.NewClient() - } - if len(name) == 0 { - name = "go.micro.store" - } return &storeService{ c: c, name: name, } } -func (c *storeService) List(ctx context.Context, in *ListRequest, opts ...client.CallOption) (Store_ListService, error) { - req := c.c.NewRequest(c.name, "Store.List", &ListRequest{}) - stream, err := c.c.Stream(ctx, req, opts...) - if err != nil { - return nil, err - } - if err := stream.Send(in); err != nil { - return nil, err - } - return &storeServiceList{stream}, nil -} - -type Store_ListService interface { - SendMsg(interface{}) error - RecvMsg(interface{}) error - Close() error - Recv() (*ListResponse, error) -} - -type storeServiceList struct { - stream client.Stream -} - -func (x *storeServiceList) Close() error { - return x.stream.Close() -} - -func (x *storeServiceList) SendMsg(m interface{}) error { - return x.stream.Send(m) -} - -func (x *storeServiceList) RecvMsg(m interface{}) error { - return x.stream.Recv(m) -} - -func (x *storeServiceList) Recv() (*ListResponse, error) { - m := new(ListResponse) - err := x.stream.Recv(m) - if err != nil { - return nil, err - } - return m, nil -} - func (c *storeService) Read(ctx context.Context, in *ReadRequest, opts ...client.CallOption) (*ReadResponse, error) { req := c.c.NewRequest(c.name, "Store.Read", in) out := new(ReadResponse) @@ -132,21 +82,70 @@ func (c *storeService) Delete(ctx context.Context, in *DeleteRequest, opts ...cl return out, nil } +func (c *storeService) List(ctx context.Context, in *ListRequest, opts ...client.CallOption) (Store_ListService, error) { + req := c.c.NewRequest(c.name, "Store.List", &ListRequest{}) + stream, err := c.c.Stream(ctx, req, opts...) + if err != nil { + return nil, err + } + if err := stream.Send(in); err != nil { + return nil, err + } + return &storeServiceList{stream}, nil +} + +type Store_ListService interface { + Context() context.Context + SendMsg(interface{}) error + RecvMsg(interface{}) error + Close() error + Recv() (*ListResponse, error) +} + +type storeServiceList struct { + stream client.Stream +} + +func (x *storeServiceList) Close() error { + return x.stream.Close() +} + +func (x *storeServiceList) Context() context.Context { + return x.stream.Context() +} + +func (x *storeServiceList) SendMsg(m interface{}) error { + return x.stream.Send(m) +} + +func (x *storeServiceList) RecvMsg(m interface{}) error { + return x.stream.Recv(m) +} + +func (x *storeServiceList) Recv() (*ListResponse, error) { + m := new(ListResponse) + err := x.stream.Recv(m) + if err != nil { + return nil, err + } + return m, nil +} + // Server API for Store service type StoreHandler interface { - List(context.Context, *ListRequest, Store_ListStream) error Read(context.Context, *ReadRequest, *ReadResponse) error Write(context.Context, *WriteRequest, *WriteResponse) error Delete(context.Context, *DeleteRequest, *DeleteResponse) error + List(context.Context, *ListRequest, Store_ListStream) error } func RegisterStoreHandler(s server.Server, hdlr StoreHandler, opts ...server.HandlerOption) error { type store interface { - List(ctx context.Context, stream server.Stream) error Read(ctx context.Context, in *ReadRequest, out *ReadResponse) error Write(ctx context.Context, in *WriteRequest, out *WriteResponse) error Delete(ctx context.Context, in *DeleteRequest, out *DeleteResponse) error + List(ctx context.Context, stream server.Stream) error } type Store struct { store @@ -159,6 +158,18 @@ type storeHandler struct { StoreHandler } +func (h *storeHandler) Read(ctx context.Context, in *ReadRequest, out *ReadResponse) error { + return h.StoreHandler.Read(ctx, in, out) +} + +func (h *storeHandler) Write(ctx context.Context, in *WriteRequest, out *WriteResponse) error { + return h.StoreHandler.Write(ctx, in, out) +} + +func (h *storeHandler) Delete(ctx context.Context, in *DeleteRequest, out *DeleteResponse) error { + return h.StoreHandler.Delete(ctx, in, out) +} + func (h *storeHandler) List(ctx context.Context, stream server.Stream) error { m := new(ListRequest) if err := stream.Recv(m); err != nil { @@ -168,6 +179,7 @@ func (h *storeHandler) List(ctx context.Context, stream server.Stream) error { } type Store_ListStream interface { + Context() context.Context SendMsg(interface{}) error RecvMsg(interface{}) error Close() error @@ -182,6 +194,10 @@ func (x *storeListStream) Close() error { return x.stream.Close() } +func (x *storeListStream) Context() context.Context { + return x.stream.Context() +} + func (x *storeListStream) SendMsg(m interface{}) error { return x.stream.Send(m) } @@ -193,15 +209,3 @@ func (x *storeListStream) RecvMsg(m interface{}) error { func (x *storeListStream) Send(m *ListResponse) error { return x.stream.Send(m) } - -func (h *storeHandler) Read(ctx context.Context, in *ReadRequest, out *ReadResponse) error { - return h.StoreHandler.Read(ctx, in, out) -} - -func (h *storeHandler) Write(ctx context.Context, in *WriteRequest, out *WriteResponse) error { - return h.StoreHandler.Write(ctx, in, out) -} - -func (h *storeHandler) Delete(ctx context.Context, in *DeleteRequest, out *DeleteResponse) error { - return h.StoreHandler.Delete(ctx, in, out) -} diff --git a/store/service/proto/store.proto b/store/service/proto/store.proto index 39bd8afc..a6462ad2 100644 --- a/store/service/proto/store.proto +++ b/store/service/proto/store.proto @@ -3,10 +3,10 @@ syntax = "proto3"; package go.micro.store; service Store { - rpc List(ListRequest) returns (stream ListResponse) {}; rpc Read(ReadRequest) returns (ReadResponse) {}; rpc Write(WriteRequest) returns (WriteResponse) {}; rpc Delete(DeleteRequest) returns (DeleteResponse) {}; + rpc List(ListRequest) returns (stream ListResponse) {}; } message Record { @@ -14,16 +14,19 @@ message Record { string key = 1; // value in the record bytes value = 2; - // timestamp in unix seconds + // time.Duration (signed int64 nanoseconds) int64 expiry = 3; } message ReadOptions { - bool prefix = 1; + bool prefix = 1; + bool suffix = 2; + uint64 limit = 3; + uint64 offset = 4; } message ReadRequest { - string key = 1; + string key = 1; ReadOptions options = 2; } @@ -31,20 +34,41 @@ message ReadResponse { repeated Record records = 1; } +message WriteOptions { + // time.Time + int64 expiry = 1; + // time.Duration + int64 ttl = 2; +} + message WriteRequest { - Record record = 1; + Record record = 1; + WriteOptions options = 2; } message WriteResponse {} +message DeleteOptions {} + message DeleteRequest { - string key = 1; + string key = 1; + DeleteOptions options = 2; } message DeleteResponse {} -message ListRequest {} +message ListOptions { + string prefix = 1; + string suffix = 2; + uint64 limit = 3; + uint64 offset = 4; +} + +message ListRequest { + ListOptions options = 1; +} message ListResponse { - repeated Record records = 1; + reserved 1; //repeated Record records = 1; + repeated string keys = 2; } diff --git a/store/service/service.go b/store/service/service.go index 713cda15..5b448a8c 100644 --- a/store/service/service.go +++ b/store/service/service.go @@ -24,6 +24,9 @@ type serviceStore struct { // Prefix to use Prefix string + // Suffix to use + Suffix string + // store service client Client pb.StoreService } @@ -56,14 +59,14 @@ func (s *serviceStore) Context() context.Context { } // Sync all the known records -func (s *serviceStore) List() ([]*store.Record, error) { +func (s *serviceStore) List(opts ...store.ListOption) ([]string, error) { stream, err := s.Client.List(s.Context(), &pb.ListRequest{}, client.WithAddress(s.Nodes...)) if err != nil { return nil, err } defer stream.Close() - var records []*store.Record + var keys []string for { rsp, err := stream.Recv() @@ -71,19 +74,15 @@ func (s *serviceStore) List() ([]*store.Record, error) { break } if err != nil { - return records, err + return keys, err } - for _, record := range rsp.Records { - records = append(records, &store.Record{ - Key: record.Key, - Value: record.Value, - Expiry: time.Duration(record.Expiry) * time.Second, - }) + for _, key := range rsp.Keys { + keys = append(keys, key) } } - return records, nil + return keys, nil } // Read a record with key @@ -117,7 +116,7 @@ func (s *serviceStore) Read(key string, opts ...store.ReadOption) ([]*store.Reco } // Write a record -func (s *serviceStore) Write(record *store.Record) error { +func (s *serviceStore) Write(record *store.Record, opts ...store.WriteOption) error { _, err := s.Client.Write(s.Context(), &pb.WriteRequest{ Record: &pb.Record{ Key: record.Key, @@ -130,7 +129,7 @@ func (s *serviceStore) Write(record *store.Record) error { } // Delete a record with key -func (s *serviceStore) Delete(key string) error { +func (s *serviceStore) Delete(key string, opts ...store.DeleteOption) error { _, err := s.Client.Delete(s.Context(), &pb.DeleteRequest{ Key: key, }, client.WithAddress(s.Nodes...)) @@ -141,6 +140,10 @@ func (s *serviceStore) String() string { return "service" } +func (s *serviceStore) Options() store.Options { + return s.options +} + // NewStore returns a new store service implementation func NewStore(opts ...store.Option) store.Store { var options store.Options diff --git a/store/store.go b/store/store.go index cf21ddee..21101cb8 100644 --- a/store/store.go +++ b/store/store.go @@ -1,4 +1,5 @@ -// Package store is an interface for distribute data storage. +// Package store is an interface for distributed data storage. +// The design document is located at https://github.com/micro/development/blob/master/design/store.md package store import ( @@ -7,66 +8,33 @@ import ( ) var ( - // ErrNotFound is returned when a Read key doesn't exist + // ErrNotFound is returned when a key doesn't exist ErrNotFound = errors.New("not found") - // Default store - DefaultStore Store = new(noop) + // DefaultStore is the memory store. + DefaultStore Store = new(noopStore) ) // Store is a data storage interface type Store interface { - // Initialise store options + // Init initialises the store. It must perform any required setup on the backing storage implementation and check that it is ready for use, returning any errors. Init(...Option) error - // List all the known records - List() ([]*Record, error) - // Read records with keys - Read(key string, opts ...ReadOption) ([]*Record, error) - // Write records - Write(*Record) error - // Delete records with keys - Delete(key string) error - // Name of the store + // Options allows you to view the current options. + Options() Options + // String returns the name of the implementation. String() string + // Read takes a single key name and optional ReadOptions. It returns matching []*Record or an error. + Read(key string, opts ...ReadOption) ([]*Record, error) + // Write() writes a record to the store, and returns an error if the record was not written. + Write(r *Record, opts ...WriteOption) error + // Delete removes the record with the corresponding key from the store. + Delete(key string, opts ...DeleteOption) error + // List returns any keys that match, or an empty list with no error if none matched. + List(opts ...ListOption) ([]string, error) } -// Record represents a data record +// Record is an item stored or retrieved from a Store type Record struct { Key string Value []byte Expiry time.Duration } - -type ReadOptions struct { - // Read key as a prefix - Prefix bool - // Read key as a suffix - Suffix bool -} - -type ReadOption func(o *ReadOptions) - -type noop struct{} - -func (n *noop) Init(...Option) error { - return nil -} - -func (n *noop) List() ([]*Record, error) { - return nil, nil -} - -func (n *noop) Read(key string, opts ...ReadOption) ([]*Record, error) { - return nil, nil -} - -func (n *noop) Write(rec *Record) error { - return nil -} - -func (n *noop) Delete(key string) error { - return nil -} - -func (n *noop) String() string { - return "noop" -} diff --git a/sync/map.go b/sync/map.go index 83736cff..edfeb6a9 100644 --- a/sync/map.go +++ b/sync/map.go @@ -90,7 +90,7 @@ func (m *syncMap) Delete(key interface{}) error { } func (m *syncMap) Iterate(fn func(key, val interface{}) error) error { - keyvals, err := m.opts.Store.List() + keyvals, err := m.opts.Store.Read("", store.ReadPrefix()) if err != nil { return err }