store: implement s3 blob store (#2005)
This commit is contained in:
ben-toogood
GitHub
4
.github/workflows/pr.yml
vendored
4
.github/workflows/pr.yml
vendored
@@ -25,6 +25,10 @@ jobs:
|
||||
id: tests
|
||||
env:
|
||||
IN_TRAVIS_CI: yes
|
||||
S3_BLOB_STORE_REGION: ${{ secrets.SCALEWAY_REGION }}
|
||||
S3_BLOB_STORE_ENDPOINT: ${{ secrets.SCALEWAY_ENDPOINT }}
|
||||
S3_BLOB_STORE_ACCESS_KEY: ${{ secrets.SCALEWAY_ACCESS_KEY }}
|
||||
S3_BLOB_STORE_SECRET_KEY: ${{ secrets.SCALEWAY_SECRET_KEY }}
|
||||
run: |
|
||||
wget -qO- https://binaries.cockroachdb.com/cockroach-v20.1.4.linux-amd64.tgz | tar xvz
|
||||
cockroach-v20.1.4.linux-amd64/cockroach start-single-node --insecure &
|
||||
|
||||
4
.github/workflows/tests.yml
vendored
4
.github/workflows/tests.yml
vendored
@@ -29,6 +29,10 @@ jobs:
|
||||
id: tests
|
||||
env:
|
||||
IN_TRAVIS_CI: yes
|
||||
S3_BLOB_STORE_REGION: ${{ secrets.SCALEWAY_REGION }}
|
||||
S3_BLOB_STORE_ENDPOINT: ${{ secrets.SCALEWAY_ENDPOINT }}
|
||||
S3_BLOB_STORE_ACCESS_KEY: ${{ secrets.SCALEWAY_ACCESS_KEY }}
|
||||
S3_BLOB_STORE_SECRET_KEY: ${{ secrets.SCALEWAY_SECRET_KEY }}
|
||||
run: |
|
||||
kubectl apply -f runtime/kubernetes/test/test.yaml
|
||||
sudo mkdir -p /var/run/secrets/kubernetes.io/serviceaccount
|
||||
|
||||
2
go.mod
2
go.mod
@@ -31,6 +31,7 @@ require (
|
||||
github.com/kr/pretty v0.2.0
|
||||
github.com/kr/text v0.2.0 // indirect
|
||||
github.com/miekg/dns v1.1.27
|
||||
github.com/minio/minio-go/v7 v7.0.5
|
||||
github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect
|
||||
github.com/oxtoacart/bpool v0.0.0-20190530202638-03653db5a59c
|
||||
github.com/patrickmn/go-cache v2.1.0+incompatible
|
||||
@@ -41,7 +42,6 @@ require (
|
||||
go.etcd.io/bbolt v1.3.5
|
||||
golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899
|
||||
golang.org/x/net v0.0.0-20200707034311-ab3426394381
|
||||
golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1 // indirect
|
||||
google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013
|
||||
google.golang.org/grpc v1.27.0
|
||||
google.golang.org/protobuf v1.25.0
|
||||
|
||||
27
go.sum
27
go.sum
@@ -119,6 +119,7 @@ github.com/go-acme/lego/v3 v3.4.0/go.mod h1:xYbLDuxq3Hy4bMUT1t9JIuz6GWIWb3m5X+Te
|
||||
github.com/go-cmd/cmd v1.0.5/go.mod h1:y8q8qlK5wQibcw63djSl/ntiHUHXHGdCkPk0j4QeW4s=
|
||||
github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q=
|
||||
github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8=
|
||||
github.com/go-ini/ini v1.44.0 h1:8+SRbfpRFlIunpSum4BEf1ClTtVjOgKzgBv9pHFkI6w=
|
||||
github.com/go-ini/ini v1.44.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8=
|
||||
github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as=
|
||||
github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE=
|
||||
@@ -180,6 +181,7 @@ github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+
|
||||
github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
|
||||
github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk=
|
||||
github.com/gophercloud/gophercloud v0.3.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8=
|
||||
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
|
||||
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY=
|
||||
github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg=
|
||||
github.com/gorilla/handlers v1.4.2 h1:0QniY0USkHQ1RGCLfKxeNHK9bkDHGRYGNDFBCS+YARg=
|
||||
@@ -212,7 +214,10 @@ github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht
|
||||
github.com/json-iterator/go v1.1.5/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
|
||||
github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU=
|
||||
github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
|
||||
github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68=
|
||||
github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4=
|
||||
github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU=
|
||||
github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo=
|
||||
github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
|
||||
github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
|
||||
github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q=
|
||||
@@ -220,6 +225,8 @@ github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQL
|
||||
github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
|
||||
github.com/klauspost/cpuid v1.2.3 h1:CCtW0xUnWGVINKvE/WWOYKdsPV6mawAtvQuSl8guwQs=
|
||||
github.com/klauspost/cpuid v1.2.3/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek=
|
||||
github.com/klauspost/cpuid v1.3.1 h1:5JNjFYYQrZeKRJ0734q51WCEEn2huer72Dc7K+R/b6s=
|
||||
github.com/klauspost/cpuid v1.3.1/go.mod h1:bYW4mA6ZgKPob1/Dlai2LviZJO7KGI3uoWLd42rAQw4=
|
||||
github.com/kolo/xmlrpc v0.0.0-20190717152603-07c4ee3fd181/go.mod h1:o03bZfuBwAXHetKXuInt4S7omeXUu62/A845kiycsSQ=
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ=
|
||||
github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s=
|
||||
@@ -245,6 +252,13 @@ github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5
|
||||
github.com/miekg/dns v1.1.15/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg=
|
||||
github.com/miekg/dns v1.1.27 h1:aEH/kqUzUxGJ/UHcEKdJY+ugH6WEzsEBBSPa8zuy1aM=
|
||||
github.com/miekg/dns v1.1.27/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM=
|
||||
github.com/minio/md5-simd v1.1.0 h1:QPfiOqlZH+Cj9teu0t9b1nTBfPbyTl16Of5MeuShdK4=
|
||||
github.com/minio/md5-simd v1.1.0/go.mod h1:XpBqgZULrMYD3R+M28PcmP0CkI7PEMzB3U77ZrKZ0Gw=
|
||||
github.com/minio/minio-go/v7 v7.0.5 h1:I2NIJ2ojwJqD/YByemC1M59e1b4FW9kS7NlOar7HPV4=
|
||||
github.com/minio/minio-go/v7 v7.0.5/go.mod h1:TA0CQCjJZHM5SJj9IjqR0NmpmQJ6bCbXifAJ3mUU6Hw=
|
||||
github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU=
|
||||
github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM=
|
||||
github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y=
|
||||
github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0=
|
||||
github.com/mitchellh/go-vnc v0.0.0-20150629162542-723ed9867aed/go.mod h1:3rdaFaCv4AyBgu5ALFM0+tSuHrBh6v692nyQe3ikrq0=
|
||||
github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y=
|
||||
@@ -313,6 +327,8 @@ github.com/rainycape/memcache v0.0.0-20150622160815-1031fa0ce2f2/go.mod h1:7tZKc
|
||||
github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4=
|
||||
github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg=
|
||||
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
|
||||
github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc=
|
||||
github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ=
|
||||
github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
|
||||
github.com/sacloud/libsacloud v1.26.1/go.mod h1:79ZwATmHLIFZIMd7sxA3LwzVy/B77uj3LDoToVTxDoQ=
|
||||
github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0=
|
||||
@@ -322,7 +338,9 @@ github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMB
|
||||
github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4=
|
||||
github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE=
|
||||
github.com/skratchdot/open-golang v0.0.0-20160302144031-75fb7ed4208c/go.mod h1:sUM3LWHvSMaG192sy56D9F7CNvL7jUJVXoqM1QKLnog=
|
||||
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM=
|
||||
github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc=
|
||||
github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a h1:pa8hGb/2YqsZKovtsgrwcDH1RZhVbTKCjLp47XpqCDs=
|
||||
github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA=
|
||||
github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME=
|
||||
@@ -447,12 +465,14 @@ golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7w
|
||||
golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1 h1:ogLJMz+qpzav7lGMh10LMvAkM/fAoGlaiiHYiFYdm80=
|
||||
golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae h1:Ih9Yo4hSPImZOpfGuA4bR/ORKTAbhZo2AbWNRCnevdo=
|
||||
golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
|
||||
golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ=
|
||||
golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs=
|
||||
golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk=
|
||||
golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k=
|
||||
golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ=
|
||||
golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
golang.org/x/time v0.0.0-20190921001708-c4c64cad1fd0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
|
||||
@@ -537,6 +557,8 @@ gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMy
|
||||
gopkg.in/h2non/gock.v1 v1.0.15/go.mod h1:sX4zAkdYX1TRGJ2JY156cFspQn4yRWn6p9EMdODlynE=
|
||||
gopkg.in/ini.v1 v1.42.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
|
||||
gopkg.in/ini.v1 v1.44.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
|
||||
gopkg.in/ini.v1 v1.57.0 h1:9unxIsFcTt4I55uWluz+UmL95q4kdJ0buvQ1ZIqVQww=
|
||||
gopkg.in/ini.v1 v1.57.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k=
|
||||
gopkg.in/ns1/ns1-go.v2 v2.0.0-20190730140822-b51389932cbc/go.mod h1:VV+3haRsgDiVLxyifmMBrBIuCWFBPYKbRssXB9z67Hw=
|
||||
gopkg.in/resty.v1 v1.9.1/go.mod h1:vo52Hzryw9PnPHcJfPsBiFW62XhNx5OczbV9y+IMpgc=
|
||||
gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo=
|
||||
@@ -548,6 +570,7 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWD
|
||||
gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74=
|
||||
gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU=
|
||||
gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI=
|
||||
gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo=
|
||||
|
||||
@@ -7,7 +7,7 @@ import (
|
||||
|
||||
var (
|
||||
// ErrMissingKey is returned when no key is passed to blob store Read / Write
|
||||
ErrMissingKey = errors.New("Missing key")
|
||||
ErrMissingKey = errors.New("missing key")
|
||||
)
|
||||
|
||||
// BlobStore is an interface for reading / writing blobs
|
||||
|
||||
@@ -126,11 +126,7 @@ func (b *blobStore) Delete(key string, opts ...store.BlobOption) error {
|
||||
// check for the namespaces bucket
|
||||
bucket := tx.Bucket([]byte(options.Namespace))
|
||||
if bucket == nil {
|
||||
return store.ErrNotFound
|
||||
}
|
||||
|
||||
if bucket.Get([]byte(key)) == nil {
|
||||
return store.ErrNotFound
|
||||
return nil
|
||||
}
|
||||
|
||||
return bucket.Delete([]byte(key))
|
||||
|
||||
@@ -60,12 +60,12 @@ func TestBlobStore(t *testing.T) {
|
||||
|
||||
t.Run("DeleteIncorrectNamespace", func(t *testing.T) {
|
||||
err := blob.Delete("hello", store.BlobNamespace("bar"))
|
||||
assert.Equal(t, store.ErrNotFound, err, "Error should be not found")
|
||||
assert.Nil(t, err, "Error should be nil")
|
||||
})
|
||||
|
||||
t.Run("DeleteCorrectNamespaceIncorrectKey", func(t *testing.T) {
|
||||
err := blob.Delete("world", store.BlobNamespace("micro"))
|
||||
assert.Equal(t, store.ErrNotFound, err, "Error should be not found")
|
||||
assert.Nil(t, err, "Error should be nil")
|
||||
})
|
||||
|
||||
t.Run("DeleteCorrectNamespace", func(t *testing.T) {
|
||||
|
||||
42
store/s3/options.go
Normal file
42
store/s3/options.go
Normal file
@@ -0,0 +1,42 @@
|
||||
package s3
|
||||
|
||||
// Options used to configure the s3 blob store
|
||||
type Options struct {
|
||||
Endpoint string
|
||||
Region string
|
||||
AccessKeyID string
|
||||
SecretAccessKey string
|
||||
Secure bool
|
||||
}
|
||||
|
||||
// Option configures one or more options
|
||||
type Option func(o *Options)
|
||||
|
||||
// Endpoint sets the endpoint option
|
||||
func Endpoint(e string) Option {
|
||||
return func(o *Options) {
|
||||
o.Endpoint = e
|
||||
}
|
||||
}
|
||||
|
||||
// Region sets the region option
|
||||
func Region(r string) Option {
|
||||
return func(o *Options) {
|
||||
o.Region = r
|
||||
}
|
||||
}
|
||||
|
||||
// Credentials sets the AccessKeyID and SecretAccessKey options
|
||||
func Credentials(id, secret string) Option {
|
||||
return func(o *Options) {
|
||||
o.AccessKeyID = id
|
||||
o.SecretAccessKey = secret
|
||||
}
|
||||
}
|
||||
|
||||
// Insecure sets the secure option to false. It is enabled by default.
|
||||
func Insecure() Option {
|
||||
return func(o *Options) {
|
||||
o.Secure = false
|
||||
}
|
||||
}
|
||||
149
store/s3/s3.go
Normal file
149
store/s3/s3.go
Normal file
@@ -0,0 +1,149 @@
|
||||
package s3
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
|
||||
"github.com/micro/go-micro/v3/store"
|
||||
"github.com/minio/minio-go/v7"
|
||||
"github.com/minio/minio-go/v7/pkg/credentials"
|
||||
"github.com/pkg/errors"
|
||||
)
|
||||
|
||||
// NewBlobStore returns an initialized s3 blob store
|
||||
func NewBlobStore(opts ...Option) (store.BlobStore, error) {
|
||||
// parse the options
|
||||
options := Options{Secure: true}
|
||||
for _, o := range opts {
|
||||
o(&options)
|
||||
}
|
||||
|
||||
// initialize minio client
|
||||
client, err := minio.New(options.Endpoint, &minio.Options{
|
||||
Creds: credentials.NewStaticV4(options.AccessKeyID, options.SecretAccessKey, ""),
|
||||
Secure: options.Secure,
|
||||
})
|
||||
if err != nil {
|
||||
return nil, errors.Wrap(err, "Error connecting to s3 blob store")
|
||||
}
|
||||
|
||||
// return the blob store
|
||||
return &s3{client, &options}, nil
|
||||
}
|
||||
|
||||
type s3 struct {
|
||||
client *minio.Client
|
||||
options *Options
|
||||
}
|
||||
|
||||
func (s *s3) Read(key string, opts ...store.BlobOption) (io.Reader, error) {
|
||||
// validate the key
|
||||
if len(key) == 0 {
|
||||
return nil, store.ErrMissingKey
|
||||
}
|
||||
|
||||
// parse the options
|
||||
var options store.BlobOptions
|
||||
for _, o := range opts {
|
||||
o(&options)
|
||||
}
|
||||
if len(options.Namespace) == 0 {
|
||||
options.Namespace = "micro"
|
||||
}
|
||||
|
||||
// lookup the object
|
||||
res, err := s.client.GetObject(
|
||||
context.TODO(), // context
|
||||
options.Namespace, // bucket name
|
||||
key, // object name
|
||||
minio.GetObjectOptions{}, // options
|
||||
)
|
||||
|
||||
// scaleway will return a 404 if the bucket doesn't exist
|
||||
if verr, ok := err.(minio.ErrorResponse); ok && verr.StatusCode == http.StatusNotFound {
|
||||
return nil, store.ErrNotFound
|
||||
} else if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// check the object info, if an error is returned the object could not be found
|
||||
_, err = res.Stat()
|
||||
if verr, ok := err.(minio.ErrorResponse); ok && verr.StatusCode == http.StatusNotFound {
|
||||
return nil, store.ErrNotFound
|
||||
} else if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// return the result
|
||||
return res, nil
|
||||
}
|
||||
|
||||
func (s *s3) Write(key string, blob io.Reader, opts ...store.BlobOption) error {
|
||||
// validate the key
|
||||
if len(key) == 0 {
|
||||
return store.ErrMissingKey
|
||||
}
|
||||
|
||||
// parse the options
|
||||
var options store.BlobOptions
|
||||
for _, o := range opts {
|
||||
o(&options)
|
||||
}
|
||||
if len(options.Namespace) == 0 {
|
||||
options.Namespace = "micro"
|
||||
}
|
||||
|
||||
// check the bucket exists, create it if not
|
||||
if exists, err := s.client.BucketExists(context.TODO(), options.Namespace); err != nil {
|
||||
return err
|
||||
} else if !exists {
|
||||
opts := minio.MakeBucketOptions{Region: s.options.Region}
|
||||
if err := s.client.MakeBucket(context.TODO(), options.Namespace, opts); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
|
||||
// get the bytes so we can determine the length
|
||||
b, err := ioutil.ReadAll(blob)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
|
||||
// create the object in the bucket
|
||||
_, err = s.client.PutObject(
|
||||
context.TODO(), // context
|
||||
options.Namespace, // bucket name
|
||||
key, // object name
|
||||
bytes.NewBuffer(b), // reader
|
||||
int64(len(b)), // length of object
|
||||
minio.PutObjectOptions{}, // options
|
||||
)
|
||||
return err
|
||||
}
|
||||
|
||||
func (s *s3) Delete(key string, opts ...store.BlobOption) error {
|
||||
// validate the key
|
||||
if len(key) == 0 {
|
||||
return store.ErrMissingKey
|
||||
}
|
||||
|
||||
// parse the options
|
||||
var options store.BlobOptions
|
||||
for _, o := range opts {
|
||||
o(&options)
|
||||
}
|
||||
if len(options.Namespace) == 0 {
|
||||
options.Namespace = "micro"
|
||||
}
|
||||
|
||||
err := s.client.RemoveObject(
|
||||
context.TODO(), // context
|
||||
options.Namespace, // bucket name
|
||||
key, // object name
|
||||
minio.RemoveObjectOptions{}, // options
|
||||
)
|
||||
return err
|
||||
}
|
||||
119
store/s3/s3_test.go
Normal file
119
store/s3/s3_test.go
Normal file
@@ -0,0 +1,119 @@
|
||||
package s3
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"io/ioutil"
|
||||
"os"
|
||||
"testing"
|
||||
|
||||
"github.com/micro/go-micro/v3/store"
|
||||
"github.com/stretchr/testify/assert"
|
||||
)
|
||||
|
||||
func TestBlobStore(t *testing.T) {
|
||||
region := os.Getenv("S3_BLOB_STORE_REGION")
|
||||
if len(region) == 0 {
|
||||
t.Skipf("Missing required config S3_BLOB_STORE_REGION")
|
||||
}
|
||||
|
||||
endpoint := os.Getenv("S3_BLOB_STORE_ENDPOINT")
|
||||
if len(endpoint) == 0 {
|
||||
t.Skipf("Missing required config S3_BLOB_STORE_ENDPOINT")
|
||||
}
|
||||
|
||||
accessKey := os.Getenv("S3_BLOB_STORE_ACCESS_KEY")
|
||||
if len(accessKey) == 0 {
|
||||
t.Skipf("Missing required config S3_BLOB_STORE_ACCESS_KEY")
|
||||
}
|
||||
|
||||
secretKey := os.Getenv("S3_BLOB_STORE_SECRET_KEY")
|
||||
if len(secretKey) == 0 {
|
||||
t.Skipf("Missing required config S3_BLOB_STORE_SECRET_KEY")
|
||||
}
|
||||
|
||||
blob, err := NewBlobStore(
|
||||
Region(region),
|
||||
Endpoint(endpoint),
|
||||
Credentials(accessKey, secretKey),
|
||||
)
|
||||
assert.NotNilf(t, blob, "Blob should not be nil")
|
||||
assert.Nilf(t, err, "Error should be nil")
|
||||
if err != nil {
|
||||
return
|
||||
}
|
||||
|
||||
t.Run("ReadMissingKey", func(t *testing.T) {
|
||||
res, err := blob.Read("")
|
||||
assert.Equal(t, store.ErrMissingKey, err, "Error should be missing key")
|
||||
assert.Nil(t, res, "Result should be nil")
|
||||
})
|
||||
|
||||
t.Run("ReadNotFound", func(t *testing.T) {
|
||||
res, err := blob.Read("foo")
|
||||
assert.Equal(t, store.ErrNotFound, err, "Error should be not found")
|
||||
assert.Nil(t, res, "Result should be nil")
|
||||
})
|
||||
|
||||
t.Run("WriteMissingKey", func(t *testing.T) {
|
||||
buf := bytes.NewBuffer([]byte("HelloWorld"))
|
||||
err := blob.Write("", buf)
|
||||
assert.Equal(t, store.ErrMissingKey, err, "Error should be missing key")
|
||||
})
|
||||
|
||||
t.Run("WriteValid", func(t *testing.T) {
|
||||
buf := bytes.NewBuffer([]byte("world"))
|
||||
err := blob.Write("hello", buf)
|
||||
assert.Nilf(t, err, "Error should be nil")
|
||||
})
|
||||
|
||||
t.Run("ReadValid", func(t *testing.T) {
|
||||
val, err := blob.Read("hello")
|
||||
assert.Nilf(t, err, "Error should be nil")
|
||||
assert.NotNilf(t, val, "Value should not be nil")
|
||||
|
||||
if val != nil {
|
||||
bytes, err := ioutil.ReadAll(val)
|
||||
assert.Nilf(t, err, "Error should be nil")
|
||||
assert.Equal(t, "world", string(bytes), "Value should be world")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("ReadIncorrectNamespace", func(t *testing.T) {
|
||||
val, err := blob.Read("hello", store.BlobNamespace("bar"))
|
||||
assert.Equal(t, store.ErrNotFound, err, "Error should be not found")
|
||||
assert.Nil(t, val, "Value should be nil")
|
||||
})
|
||||
|
||||
t.Run("ReadCorrectNamespace", func(t *testing.T) {
|
||||
val, err := blob.Read("hello", store.BlobNamespace("micro"))
|
||||
assert.Nil(t, err, "Error should be nil")
|
||||
assert.NotNilf(t, val, "Value should not be nil")
|
||||
|
||||
if val != nil {
|
||||
bytes, err := ioutil.ReadAll(val)
|
||||
assert.Nilf(t, err, "Error should be nil")
|
||||
assert.Equal(t, "world", string(bytes), "Value should be world")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("DeleteIncorrectNamespace", func(t *testing.T) {
|
||||
err := blob.Delete("hello", store.BlobNamespace("bar"))
|
||||
assert.Nil(t, err, "Error should be nil")
|
||||
})
|
||||
|
||||
t.Run("DeleteCorrectNamespaceIncorrectKey", func(t *testing.T) {
|
||||
err := blob.Delete("world", store.BlobNamespace("micro"))
|
||||
assert.Nil(t, err, "Error should be nil")
|
||||
})
|
||||
|
||||
t.Run("DeleteCorrectNamespace", func(t *testing.T) {
|
||||
err := blob.Delete("hello", store.BlobNamespace("micro"))
|
||||
assert.Nil(t, err, "Error should be nil")
|
||||
})
|
||||
|
||||
t.Run("ReadDeletedKey", func(t *testing.T) {
|
||||
res, err := blob.Read("hello", store.BlobNamespace("micro"))
|
||||
assert.Equal(t, store.ErrNotFound, err, "Error should be not found")
|
||||
assert.Nil(t, res, "Result should be nil")
|
||||
})
|
||||
}
|
||||
Reference in New Issue
Block a user