From 01d9bb6a59c05039a5a37e297c3bf9ea0b46d0d4 Mon Sep 17 00:00:00 2001 From: ben-toogood Date: Tue, 15 Sep 2020 14:05:10 +0100 Subject: [PATCH] store: add blob interface with file implementation (#2004) --- blob.go | 138 +++++++++++++++++++++++++++++++++++++++++++++++++++ blob_test.go | 81 ++++++++++++++++++++++++++++++ 2 files changed, 219 insertions(+) create mode 100644 blob.go create mode 100644 blob_test.go diff --git a/blob.go b/blob.go new file mode 100644 index 0000000..4022cb6 --- /dev/null +++ b/blob.go @@ -0,0 +1,138 @@ +package file + +import ( + "bytes" + "io" + "io/ioutil" + "os" + "path/filepath" + "time" + + "github.com/micro/go-micro/v3/store" + "github.com/pkg/errors" + bolt "go.etcd.io/bbolt" +) + +// NewBlobStore returns a blob file store +func NewBlobStore() (store.BlobStore, error) { + // ensure the parent directory exists + os.MkdirAll(DefaultDir, 0700) + + // open the connection to the database + dbPath := filepath.Join(DefaultDir, "micro.db") + db, err := bolt.Open(dbPath, 0700, &bolt.Options{Timeout: 5 * time.Second}) + if err != nil { + return nil, errors.Wrap(err, "Error connecting to database") + } + + // return the blob store + return &blobStore{db}, nil +} + +type blobStore struct { + db *bolt.DB +} + +func (b *blobStore) Read(key string, opts ...store.BlobOption) (io.Reader, error) { + // validate the key + if len(key) == 0 { + return nil, store.ErrMissingKey + } + + // parse the options + var options store.BlobOptions + for _, o := range opts { + o(&options) + } + if len(options.Namespace) == 0 { + options.Namespace = "micro" + } + + // execute the transaction + var value []byte + readValue := func(tx *bolt.Tx) error { + // check for the namespaces bucket + bucket := tx.Bucket([]byte(options.Namespace)) + if bucket == nil { + return store.ErrNotFound + } + + // look for the blob within the bucket + value = bucket.Get([]byte(key)) + if value == nil { + return store.ErrNotFound + } + + return nil + } + if err := b.db.View(readValue); err != nil { + return nil, err + } + + // return the blob + return bytes.NewBuffer(value), nil +} + +func (b *blobStore) Write(key string, blob io.Reader, opts ...store.BlobOption) error { + // validate the key + if len(key) == 0 { + return store.ErrMissingKey + } + + // parse the options + var options store.BlobOptions + for _, o := range opts { + o(&options) + } + if len(options.Namespace) == 0 { + options.Namespace = "micro" + } + + // execute the transaction + return b.db.Update(func(tx *bolt.Tx) error { + // create the bucket + bucket, err := tx.CreateBucketIfNotExists([]byte(options.Namespace)) + if err != nil { + return err + } + + // write to the bucket + value, err := ioutil.ReadAll(blob) + if err != nil { + return err + } + + return bucket.Put([]byte(key), value) + }) +} + +func (b *blobStore) Delete(key string, opts ...store.BlobOption) error { + // validate the key + if len(key) == 0 { + return store.ErrMissingKey + } + + // parse the options + var options store.BlobOptions + for _, o := range opts { + o(&options) + } + if len(options.Namespace) == 0 { + options.Namespace = "micro" + } + + // execute the transaction + return b.db.Update(func(tx *bolt.Tx) error { + // check for the namespaces bucket + bucket := tx.Bucket([]byte(options.Namespace)) + if bucket == nil { + return store.ErrNotFound + } + + if bucket.Get([]byte(key)) == nil { + return store.ErrNotFound + } + + return bucket.Delete([]byte(key)) + }) +} diff --git a/blob_test.go b/blob_test.go new file mode 100644 index 0000000..34fcab0 --- /dev/null +++ b/blob_test.go @@ -0,0 +1,81 @@ +package file + +import ( + "bytes" + "io/ioutil" + "testing" + + "github.com/micro/go-micro/v3/store" + "github.com/stretchr/testify/assert" +) + +func TestBlobStore(t *testing.T) { + blob, err := NewBlobStore() + assert.NotNilf(t, blob, "Blob should not be nil") + assert.Nilf(t, err, "Error should be nil") + + t.Run("ReadMissingKey", func(t *testing.T) { + res, err := blob.Read("") + assert.Equal(t, store.ErrMissingKey, err, "Error should be missing key") + assert.Nil(t, res, "Result should be nil") + }) + + t.Run("ReadNotFound", func(t *testing.T) { + res, err := blob.Read("foo") + assert.Equal(t, store.ErrNotFound, err, "Error should be not found") + assert.Nil(t, res, "Result should be nil") + }) + + t.Run("WriteMissingKey", func(t *testing.T) { + buf := bytes.NewBuffer([]byte("HelloWorld")) + err := blob.Write("", buf) + assert.Equal(t, store.ErrMissingKey, err, "Error should be missing key") + }) + + t.Run("WriteValid", func(t *testing.T) { + buf := bytes.NewBuffer([]byte("world")) + err := blob.Write("hello", buf) + assert.Nilf(t, err, "Error should be nil") + }) + + t.Run("ReadValid", func(t *testing.T) { + val, err := blob.Read("hello") + bytes, _ := ioutil.ReadAll(val) + assert.Nilf(t, err, "Error should be nil") + assert.Equal(t, string(bytes), "world", "Value should be world") + }) + + t.Run("ReadIncorrectNamespace", func(t *testing.T) { + val, err := blob.Read("hello", store.BlobNamespace("bar")) + assert.Equal(t, store.ErrNotFound, err, "Error should be not found") + assert.Nil(t, val, "Value should be nil") + }) + + t.Run("ReadCorrectNamespace", func(t *testing.T) { + val, err := blob.Read("hello", store.BlobNamespace("micro")) + bytes, _ := ioutil.ReadAll(val) + assert.Nil(t, err, "Error should be nil") + assert.Equal(t, string(bytes), "world", "Value should be world") + }) + + t.Run("DeleteIncorrectNamespace", func(t *testing.T) { + err := blob.Delete("hello", store.BlobNamespace("bar")) + assert.Equal(t, store.ErrNotFound, err, "Error should be not found") + }) + + t.Run("DeleteCorrectNamespaceIncorrectKey", func(t *testing.T) { + err := blob.Delete("world", store.BlobNamespace("micro")) + assert.Equal(t, store.ErrNotFound, err, "Error should be not found") + }) + + t.Run("DeleteCorrectNamespace", func(t *testing.T) { + err := blob.Delete("hello", store.BlobNamespace("micro")) + assert.Nil(t, err, "Error should be nil") + }) + + t.Run("ReadDeletedKey", func(t *testing.T) { + res, err := blob.Read("hello", store.BlobNamespace("micro")) + assert.Equal(t, store.ErrNotFound, err, "Error should be not found") + assert.Nil(t, res, "Result should be nil") + }) +}