From 35349bd31343c70d5a8c14b7e74d4c900e72abe6 Mon Sep 17 00:00:00 2001 From: ben-toogood Date: Tue, 15 Sep 2020 17:09:40 +0100 Subject: [PATCH] store: implement s3 blob store (#2005) --- .github/workflows/pr.yml | 4 + .github/workflows/tests.yml | 4 + go.mod | 2 +- go.sum | 27 ++++++- store/blob.go | 2 +- store/file/blob.go | 6 +- store/file/blob_test.go | 4 +- store/s3/options.go | 42 ++++++++++ store/s3/s3.go | 149 ++++++++++++++++++++++++++++++++++++ store/s3/s3_test.go | 119 ++++++++++++++++++++++++++++ 10 files changed, 348 insertions(+), 11 deletions(-) create mode 100644 store/s3/options.go create mode 100644 store/s3/s3.go create mode 100644 store/s3/s3_test.go diff --git a/.github/workflows/pr.yml b/.github/workflows/pr.yml index edf62196..7412821e 100644 --- a/.github/workflows/pr.yml +++ b/.github/workflows/pr.yml @@ -25,6 +25,10 @@ jobs: id: tests env: IN_TRAVIS_CI: yes + S3_BLOB_STORE_REGION: ${{ secrets.SCALEWAY_REGION }} + S3_BLOB_STORE_ENDPOINT: ${{ secrets.SCALEWAY_ENDPOINT }} + S3_BLOB_STORE_ACCESS_KEY: ${{ secrets.SCALEWAY_ACCESS_KEY }} + S3_BLOB_STORE_SECRET_KEY: ${{ secrets.SCALEWAY_SECRET_KEY }} run: | wget -qO- https://binaries.cockroachdb.com/cockroach-v20.1.4.linux-amd64.tgz | tar xvz cockroach-v20.1.4.linux-amd64/cockroach start-single-node --insecure & diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 53458695..6acba583 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -29,6 +29,10 @@ jobs: id: tests env: IN_TRAVIS_CI: yes + S3_BLOB_STORE_REGION: ${{ secrets.SCALEWAY_REGION }} + S3_BLOB_STORE_ENDPOINT: ${{ secrets.SCALEWAY_ENDPOINT }} + S3_BLOB_STORE_ACCESS_KEY: ${{ secrets.SCALEWAY_ACCESS_KEY }} + S3_BLOB_STORE_SECRET_KEY: ${{ secrets.SCALEWAY_SECRET_KEY }} run: | kubectl apply -f runtime/kubernetes/test/test.yaml sudo mkdir -p /var/run/secrets/kubernetes.io/serviceaccount diff --git a/go.mod b/go.mod index 01024123..43548676 100644 --- a/go.mod +++ b/go.mod @@ -31,6 +31,7 @@ require ( github.com/kr/pretty v0.2.0 github.com/kr/text v0.2.0 // indirect github.com/miekg/dns v1.1.27 + github.com/minio/minio-go/v7 v7.0.5 github.com/niemeyer/pretty v0.0.0-20200227124842-a10e7caefd8e // indirect github.com/oxtoacart/bpool v0.0.0-20190530202638-03653db5a59c github.com/patrickmn/go-cache v2.1.0+incompatible @@ -41,7 +42,6 @@ require ( go.etcd.io/bbolt v1.3.5 golang.org/x/crypto v0.0.0-20200709230013-948cd5f35899 golang.org/x/net v0.0.0-20200707034311-ab3426394381 - golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1 // indirect google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013 google.golang.org/grpc v1.27.0 google.golang.org/protobuf v1.25.0 diff --git a/go.sum b/go.sum index 17cf09cc..bcb4c592 100644 --- a/go.sum +++ b/go.sum @@ -119,6 +119,7 @@ github.com/go-acme/lego/v3 v3.4.0/go.mod h1:xYbLDuxq3Hy4bMUT1t9JIuz6GWIWb3m5X+Te github.com/go-cmd/cmd v1.0.5/go.mod h1:y8q8qlK5wQibcw63djSl/ntiHUHXHGdCkPk0j4QeW4s= github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-ini/ini v1.44.0 h1:8+SRbfpRFlIunpSum4BEf1ClTtVjOgKzgBv9pHFkI6w= github.com/go-ini/ini v1.44.0/go.mod h1:ByCAeIL28uOIIG0E3PJtZPDL8WnHpFKFOtgjp+3Ies8= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= @@ -180,6 +181,7 @@ github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= github.com/gophercloud/gophercloud v0.3.0/go.mod h1:vxM41WHh5uqHVBMZHzuwNOHh8XEoIEcSTewFxm1c5g8= +github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= github.com/gorilla/handlers v1.4.2 h1:0QniY0USkHQ1RGCLfKxeNHK9bkDHGRYGNDFBCS+YARg= @@ -212,7 +214,10 @@ github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht github.com/json-iterator/go v1.1.5/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.10 h1:Kz6Cvnvv2wGdaG/V8yMvfkmNiXq9Ya2KUv4rouJJr68= +github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= +github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w= github.com/kisielk/errcheck v1.1.0/go.mod h1:EZBBE59ingxPouuu3KfxchcWSUPOHkagtvWXihfKN4Q= @@ -220,6 +225,8 @@ github.com/kisielk/errcheck v1.2.0/go.mod h1:/BMXB+zMLi60iA8Vv6Ksmxu/1UDYcXs4uQL github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/cpuid v1.2.3 h1:CCtW0xUnWGVINKvE/WWOYKdsPV6mawAtvQuSl8guwQs= github.com/klauspost/cpuid v1.2.3/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= +github.com/klauspost/cpuid v1.3.1 h1:5JNjFYYQrZeKRJ0734q51WCEEn2huer72Dc7K+R/b6s= +github.com/klauspost/cpuid v1.3.1/go.mod h1:bYW4mA6ZgKPob1/Dlai2LviZJO7KGI3uoWLd42rAQw4= github.com/kolo/xmlrpc v0.0.0-20190717152603-07c4ee3fd181/go.mod h1:o03bZfuBwAXHetKXuInt4S7omeXUu62/A845kiycsSQ= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.2 h1:DB17ag19krx9CFsz4o3enTrPXyIXCl+2iCXH/aMAp9s= @@ -245,6 +252,13 @@ github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5 github.com/miekg/dns v1.1.15/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.27 h1:aEH/kqUzUxGJ/UHcEKdJY+ugH6WEzsEBBSPa8zuy1aM= github.com/miekg/dns v1.1.27/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= +github.com/minio/md5-simd v1.1.0 h1:QPfiOqlZH+Cj9teu0t9b1nTBfPbyTl16Of5MeuShdK4= +github.com/minio/md5-simd v1.1.0/go.mod h1:XpBqgZULrMYD3R+M28PcmP0CkI7PEMzB3U77ZrKZ0Gw= +github.com/minio/minio-go/v7 v7.0.5 h1:I2NIJ2ojwJqD/YByemC1M59e1b4FW9kS7NlOar7HPV4= +github.com/minio/minio-go/v7 v7.0.5/go.mod h1:TA0CQCjJZHM5SJj9IjqR0NmpmQJ6bCbXifAJ3mUU6Hw= +github.com/minio/sha256-simd v0.1.1 h1:5QHSlgo3nt5yKOJrC7W8w7X+NFl8cMPZm96iu8kKUJU= +github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= +github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-vnc v0.0.0-20150629162542-723ed9867aed/go.mod h1:3rdaFaCv4AyBgu5ALFM0+tSuHrBh6v692nyQe3ikrq0= github.com/mitchellh/mapstructure v1.1.2/go.mod h1:FVVH3fgwuzCH5S8UJGiWEs2h04kUh9fWfEaFds41c1Y= @@ -313,6 +327,8 @@ github.com/rainycape/memcache v0.0.0-20150622160815-1031fa0ce2f2/go.mod h1:7tZKc github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rs/xid v1.2.1 h1:mhH9Nq+C1fY2l1XIpgxIiUOfNpRBYH1kKcr+qfKgjRc= +github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/sacloud/libsacloud v1.26.1/go.mod h1:79ZwATmHLIFZIMd7sxA3LwzVy/B77uj3LDoToVTxDoQ= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= @@ -322,7 +338,9 @@ github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMB github.com/sirupsen/logrus v1.4.2 h1:SPIRibHv4MatM3XXNO2BJeFLZwZ2LvZgfQ5+UNI2im4= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/skratchdot/open-golang v0.0.0-20160302144031-75fb7ed4208c/go.mod h1:sUM3LWHvSMaG192sy56D9F7CNvL7jUJVXoqM1QKLnog= +github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d h1:zE9ykElWQ6/NYmHa3jpm/yHnI4xSofP+UP6SpjHcSeM= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a h1:pa8hGb/2YqsZKovtsgrwcDH1RZhVbTKCjLp47XpqCDs= github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= @@ -447,12 +465,14 @@ golang.org/x/sys v0.0.0-20190924154521-2837fb4f24fe/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20191005200804-aed5e4c7ecf9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200202164722-d101bd2416d5/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1 h1:ogLJMz+qpzav7lGMh10LMvAkM/fAoGlaiiHYiFYdm80= -golang.org/x/sys v0.0.0-20200615200032-f1bc736245b1/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae h1:Ih9Yo4hSPImZOpfGuA4bR/ORKTAbhZo2AbWNRCnevdo= +golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2 h1:tW2bmiBqwgJj/UpqtC8EpXEZVYOwU0yG4iWbprSVAcs= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/text v0.3.3 h1:cokOdA+Jmi5PJGXLlLllQSgYigAEfHXJAERHVMaCc2k= +golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190921001708-c4c64cad1fd0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -537,6 +557,8 @@ gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMy gopkg.in/h2non/gock.v1 v1.0.15/go.mod h1:sX4zAkdYX1TRGJ2JY156cFspQn4yRWn6p9EMdODlynE= gopkg.in/ini.v1 v1.42.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ini.v1 v1.44.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/ini.v1 v1.57.0 h1:9unxIsFcTt4I55uWluz+UmL95q4kdJ0buvQ1ZIqVQww= +gopkg.in/ini.v1 v1.57.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/ns1/ns1-go.v2 v2.0.0-20190730140822-b51389932cbc/go.mod h1:VV+3haRsgDiVLxyifmMBrBIuCWFBPYKbRssXB9z67Hw= gopkg.in/resty.v1 v1.9.1/go.mod h1:vo52Hzryw9PnPHcJfPsBiFW62XhNx5OczbV9y+IMpgc= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= @@ -548,6 +570,7 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWD gopkg.in/yaml.v2 v2.0.0-20170812160011-eb3733d160e7/go.mod h1:JAlM8MvJe8wmxCU4Bli9HhUf9+ttbYbLASfIpnQbh74= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0 h1:clyUAQHOM3G0M3f5vQj7LuJrETvjVot3Z5el9nffUtU= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gotest.tools v2.2.0+incompatible h1:VsBPFP1AI068pPrMxtb/S8Zkgf9xEmTLJjfM+P5UIEo= diff --git a/store/blob.go b/store/blob.go index 107b7a9b..2cb95f91 100644 --- a/store/blob.go +++ b/store/blob.go @@ -7,7 +7,7 @@ import ( var ( // ErrMissingKey is returned when no key is passed to blob store Read / Write - ErrMissingKey = errors.New("Missing key") + ErrMissingKey = errors.New("missing key") ) // BlobStore is an interface for reading / writing blobs diff --git a/store/file/blob.go b/store/file/blob.go index 4022cb65..9f2ee511 100644 --- a/store/file/blob.go +++ b/store/file/blob.go @@ -126,11 +126,7 @@ func (b *blobStore) Delete(key string, opts ...store.BlobOption) error { // check for the namespaces bucket bucket := tx.Bucket([]byte(options.Namespace)) if bucket == nil { - return store.ErrNotFound - } - - if bucket.Get([]byte(key)) == nil { - return store.ErrNotFound + return nil } return bucket.Delete([]byte(key)) diff --git a/store/file/blob_test.go b/store/file/blob_test.go index 34fcab0f..815b9816 100644 --- a/store/file/blob_test.go +++ b/store/file/blob_test.go @@ -60,12 +60,12 @@ func TestBlobStore(t *testing.T) { t.Run("DeleteIncorrectNamespace", func(t *testing.T) { err := blob.Delete("hello", store.BlobNamespace("bar")) - assert.Equal(t, store.ErrNotFound, err, "Error should be not found") + assert.Nil(t, err, "Error should be nil") }) t.Run("DeleteCorrectNamespaceIncorrectKey", func(t *testing.T) { err := blob.Delete("world", store.BlobNamespace("micro")) - assert.Equal(t, store.ErrNotFound, err, "Error should be not found") + assert.Nil(t, err, "Error should be nil") }) t.Run("DeleteCorrectNamespace", func(t *testing.T) { diff --git a/store/s3/options.go b/store/s3/options.go new file mode 100644 index 00000000..266bee90 --- /dev/null +++ b/store/s3/options.go @@ -0,0 +1,42 @@ +package s3 + +// Options used to configure the s3 blob store +type Options struct { + Endpoint string + Region string + AccessKeyID string + SecretAccessKey string + Secure bool +} + +// Option configures one or more options +type Option func(o *Options) + +// Endpoint sets the endpoint option +func Endpoint(e string) Option { + return func(o *Options) { + o.Endpoint = e + } +} + +// Region sets the region option +func Region(r string) Option { + return func(o *Options) { + o.Region = r + } +} + +// Credentials sets the AccessKeyID and SecretAccessKey options +func Credentials(id, secret string) Option { + return func(o *Options) { + o.AccessKeyID = id + o.SecretAccessKey = secret + } +} + +// Insecure sets the secure option to false. It is enabled by default. +func Insecure() Option { + return func(o *Options) { + o.Secure = false + } +} diff --git a/store/s3/s3.go b/store/s3/s3.go new file mode 100644 index 00000000..52fd69bf --- /dev/null +++ b/store/s3/s3.go @@ -0,0 +1,149 @@ +package s3 + +import ( + "bytes" + "context" + "io" + "io/ioutil" + "net/http" + + "github.com/micro/go-micro/v3/store" + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" + "github.com/pkg/errors" +) + +// NewBlobStore returns an initialized s3 blob store +func NewBlobStore(opts ...Option) (store.BlobStore, error) { + // parse the options + options := Options{Secure: true} + for _, o := range opts { + o(&options) + } + + // initialize minio client + client, err := minio.New(options.Endpoint, &minio.Options{ + Creds: credentials.NewStaticV4(options.AccessKeyID, options.SecretAccessKey, ""), + Secure: options.Secure, + }) + if err != nil { + return nil, errors.Wrap(err, "Error connecting to s3 blob store") + } + + // return the blob store + return &s3{client, &options}, nil +} + +type s3 struct { + client *minio.Client + options *Options +} + +func (s *s3) Read(key string, opts ...store.BlobOption) (io.Reader, error) { + // validate the key + if len(key) == 0 { + return nil, store.ErrMissingKey + } + + // parse the options + var options store.BlobOptions + for _, o := range opts { + o(&options) + } + if len(options.Namespace) == 0 { + options.Namespace = "micro" + } + + // lookup the object + res, err := s.client.GetObject( + context.TODO(), // context + options.Namespace, // bucket name + key, // object name + minio.GetObjectOptions{}, // options + ) + + // scaleway will return a 404 if the bucket doesn't exist + if verr, ok := err.(minio.ErrorResponse); ok && verr.StatusCode == http.StatusNotFound { + return nil, store.ErrNotFound + } else if err != nil { + return nil, err + } + + // check the object info, if an error is returned the object could not be found + _, err = res.Stat() + if verr, ok := err.(minio.ErrorResponse); ok && verr.StatusCode == http.StatusNotFound { + return nil, store.ErrNotFound + } else if err != nil { + return nil, err + } + + // return the result + return res, nil +} + +func (s *s3) Write(key string, blob io.Reader, opts ...store.BlobOption) error { + // validate the key + if len(key) == 0 { + return store.ErrMissingKey + } + + // parse the options + var options store.BlobOptions + for _, o := range opts { + o(&options) + } + if len(options.Namespace) == 0 { + options.Namespace = "micro" + } + + // check the bucket exists, create it if not + if exists, err := s.client.BucketExists(context.TODO(), options.Namespace); err != nil { + return err + } else if !exists { + opts := minio.MakeBucketOptions{Region: s.options.Region} + if err := s.client.MakeBucket(context.TODO(), options.Namespace, opts); err != nil { + return err + } + } + + // get the bytes so we can determine the length + b, err := ioutil.ReadAll(blob) + if err != nil { + return err + } + + // create the object in the bucket + _, err = s.client.PutObject( + context.TODO(), // context + options.Namespace, // bucket name + key, // object name + bytes.NewBuffer(b), // reader + int64(len(b)), // length of object + minio.PutObjectOptions{}, // options + ) + return err +} + +func (s *s3) Delete(key string, opts ...store.BlobOption) error { + // validate the key + if len(key) == 0 { + return store.ErrMissingKey + } + + // parse the options + var options store.BlobOptions + for _, o := range opts { + o(&options) + } + if len(options.Namespace) == 0 { + options.Namespace = "micro" + } + + err := s.client.RemoveObject( + context.TODO(), // context + options.Namespace, // bucket name + key, // object name + minio.RemoveObjectOptions{}, // options + ) + return err +} diff --git a/store/s3/s3_test.go b/store/s3/s3_test.go new file mode 100644 index 00000000..c736dce1 --- /dev/null +++ b/store/s3/s3_test.go @@ -0,0 +1,119 @@ +package s3 + +import ( + "bytes" + "io/ioutil" + "os" + "testing" + + "github.com/micro/go-micro/v3/store" + "github.com/stretchr/testify/assert" +) + +func TestBlobStore(t *testing.T) { + region := os.Getenv("S3_BLOB_STORE_REGION") + if len(region) == 0 { + t.Skipf("Missing required config S3_BLOB_STORE_REGION") + } + + endpoint := os.Getenv("S3_BLOB_STORE_ENDPOINT") + if len(endpoint) == 0 { + t.Skipf("Missing required config S3_BLOB_STORE_ENDPOINT") + } + + accessKey := os.Getenv("S3_BLOB_STORE_ACCESS_KEY") + if len(accessKey) == 0 { + t.Skipf("Missing required config S3_BLOB_STORE_ACCESS_KEY") + } + + secretKey := os.Getenv("S3_BLOB_STORE_SECRET_KEY") + if len(secretKey) == 0 { + t.Skipf("Missing required config S3_BLOB_STORE_SECRET_KEY") + } + + blob, err := NewBlobStore( + Region(region), + Endpoint(endpoint), + Credentials(accessKey, secretKey), + ) + assert.NotNilf(t, blob, "Blob should not be nil") + assert.Nilf(t, err, "Error should be nil") + if err != nil { + return + } + + t.Run("ReadMissingKey", func(t *testing.T) { + res, err := blob.Read("") + assert.Equal(t, store.ErrMissingKey, err, "Error should be missing key") + assert.Nil(t, res, "Result should be nil") + }) + + t.Run("ReadNotFound", func(t *testing.T) { + res, err := blob.Read("foo") + assert.Equal(t, store.ErrNotFound, err, "Error should be not found") + assert.Nil(t, res, "Result should be nil") + }) + + t.Run("WriteMissingKey", func(t *testing.T) { + buf := bytes.NewBuffer([]byte("HelloWorld")) + err := blob.Write("", buf) + assert.Equal(t, store.ErrMissingKey, err, "Error should be missing key") + }) + + t.Run("WriteValid", func(t *testing.T) { + buf := bytes.NewBuffer([]byte("world")) + err := blob.Write("hello", buf) + assert.Nilf(t, err, "Error should be nil") + }) + + t.Run("ReadValid", func(t *testing.T) { + val, err := blob.Read("hello") + assert.Nilf(t, err, "Error should be nil") + assert.NotNilf(t, val, "Value should not be nil") + + if val != nil { + bytes, err := ioutil.ReadAll(val) + assert.Nilf(t, err, "Error should be nil") + assert.Equal(t, "world", string(bytes), "Value should be world") + } + }) + + t.Run("ReadIncorrectNamespace", func(t *testing.T) { + val, err := blob.Read("hello", store.BlobNamespace("bar")) + assert.Equal(t, store.ErrNotFound, err, "Error should be not found") + assert.Nil(t, val, "Value should be nil") + }) + + t.Run("ReadCorrectNamespace", func(t *testing.T) { + val, err := blob.Read("hello", store.BlobNamespace("micro")) + assert.Nil(t, err, "Error should be nil") + assert.NotNilf(t, val, "Value should not be nil") + + if val != nil { + bytes, err := ioutil.ReadAll(val) + assert.Nilf(t, err, "Error should be nil") + assert.Equal(t, "world", string(bytes), "Value should be world") + } + }) + + t.Run("DeleteIncorrectNamespace", func(t *testing.T) { + err := blob.Delete("hello", store.BlobNamespace("bar")) + assert.Nil(t, err, "Error should be nil") + }) + + t.Run("DeleteCorrectNamespaceIncorrectKey", func(t *testing.T) { + err := blob.Delete("world", store.BlobNamespace("micro")) + assert.Nil(t, err, "Error should be nil") + }) + + t.Run("DeleteCorrectNamespace", func(t *testing.T) { + err := blob.Delete("hello", store.BlobNamespace("micro")) + assert.Nil(t, err, "Error should be nil") + }) + + t.Run("ReadDeletedKey", func(t *testing.T) { + res, err := blob.Read("hello", store.BlobNamespace("micro")) + assert.Equal(t, store.ErrNotFound, err, "Error should be not found") + assert.Nil(t, res, "Result should be nil") + }) +}