initial import

Signed-off-by: Vasiliy Tolstov <v.tolstov@selfip.ru>

LICENSE

@@ -0,0 +1,21 @@
+The MIT License (MIT)
+
+Copyright (c) 2016-2018 sdstack and other authors.
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -0,0 +1,3 @@
+storage
+=======
+

api/api.go

@@ -0,0 +1,58 @@
+package api
+
+/*
+import (
+	"fmt"
+	"strings"
+
+	"github.com/sdstack/storage/kv"
+)
+
+var apiTypes map[string]Api
+
+func init() {
+	apiTypes = make(map[string]Api)
+}
+
+func RegisterApi(engine string, api Api) {
+	apiTypes[engine] = proxy
+}
+
+// Proxy represents proxy interface
+type Proxy interface {
+	Start() error
+	Stop() error
+	Configure(*kv.KV, interface{}) error
+	//	Format() error
+	//	Check() error
+	//	Recover() error
+	//	Info() (*Info, error)
+	//	Snapshot() error
+	//	Reweight() error
+	//	Members() []Member
+}
+
+func New(ptype string, cfg interface{}, engine *kv.KV) (Proxy, error) {
+	var err error
+
+	proxy, ok := proxyTypes[ptype]
+	if !ok {
+		return nil, fmt.Errorf("unknown proxy type %s, only %s supported", ptype, strings.Join(ProxyTypes(), ","))
+	}
+
+	err = proxy.Configure(engine, cfg)
+	if err != nil {
+		return nil, err
+	}
+
+	return proxy, nil
+}
+
+func ProxyTypes() []string {
+	var ptypes []string
+	for ptype, _ := range proxyTypes {
+		ptypes = append(ptypes, ptype)
+	}
+	return ptypes
+}
+*/

backend/backend.go

@@ -0,0 +1,87 @@
+package backend
+
+import (
+	"errors"
+	"fmt"
+	"io"
+	"strings"
+)
+
+/*
+You could do "files" as variable length blocks, where you have a list of blocks needed to reconstruct. To insert data you create a new block, keep the original, but specify that you only need parts of the data.
+A file is a list of blocks. When you insert new data, you create a new block, and adjust start/end of affected blocks.
+This allows you to insert data at the same speed as end-of-file writes. You can add some lazy cleanup, that removes unused parts of blocks and recompresses them, but the important part is that you do not need to do that when the file is updated.
+There will only be a read overhead when "start" of a block is >0. That will be the same for using "ReadAt". You skip in your blocks until you get to the position you want and start decoding from there.
+If blocks are stored based on their uncompressed hash, you furthermore get a coarse high-level deduplication, though that of course makes it more difficult to figure out when a block is no longer used by any file.
+With that, you should get pretty good performance, even with block sizes up to 16-64MB.
+For each block, I would probably go with variably sized de-duplication, with a 4K average, and deflate the result @level 1. That should give you ~50-60MB/core throughput. If you need higher write throughput, you can always set deflate to level 0 (store), and leave it up to a lazy task to compress the data. That should give you about 150MB/core. Obviously you can de-duplicate/compress several blocks in parallel.
+*/
+
+/*
+type File struct {
+	Name   string
+	Size   uint64
+	Blocks []Block
+	Ring   *config.Ring
+}
+
+type Block struct {
+	Start uint64
+	Stop  uint64
+	Size  uint64
+}
+*/
+
+var (
+	ErrIO = errors.New("IO error")
+)
+
+var backendTypes map[string]Backend
+
+func init() {
+	backendTypes = make(map[string]Backend)
+}
+
+func RegisterBackend(engine string, backend Backend) {
+	backendTypes[engine] = backend
+}
+
+func New(btype string, cfg interface{}) (Backend, error) {
+	var err error
+
+	backend, ok := backendTypes[btype]
+	if !ok {
+		return nil, fmt.Errorf("unknown backend type %s. only %s supported", btype, strings.Join(BackendTypes(), ","))
+	}
+
+	if cfg == nil {
+		return backend, nil
+	}
+
+	err = backend.Init(cfg)
+	if err != nil {
+		return nil, err
+	}
+
+	return backend, nil
+}
+
+func BackendTypes() []string {
+	var btypes []string
+	for btype, _ := range backendTypes {
+		btypes = append(btypes, btype)
+	}
+	return btypes
+}
+
+type Backend interface {
+	Configure(interface{}) error
+	Init(interface{}) error
+	ReaderFrom(string, io.Reader, int64, int64, int, int) (int64, error)
+	WriterTo(string, io.Writer, int64, int64, int, int) (int64, error)
+	WriteAt(string, []byte, int64, int, int) (int, error)
+	ReadAt(string, []byte, int64, int, int) (int, error)
+	Allocate(string, int64, int, int) error
+	Remove(string, int, int) error
+	Exists(string, int, int) (bool, error)
+}

backend/block/block.go

@@ -0,0 +1,100 @@
+package block
+
+/*
+import (
+	"fmt"
+	"os"
+
+	"github.com/mitchellh/mapstructure"
+	"golang.org/x/sys/unix"
+	"github.com/sdstack/storage/backend"
+)
+
+type StoreBlock struct {
+	store []string
+}
+
+type ObjectBlock struct {
+	fs *StoreBlock
+	fp *os.File
+}
+
+func init() {
+	backend.RegisterBackend("block", &StoreBlock{})
+}
+func (s *StoreBlock) Init(data interface{}) error {
+	var err error
+
+	err = mapstructure.Decode(data, &s.store)
+	if err != nil {
+		return err
+	}
+	fmt.Printf("%#+v\n", s)
+	return nil
+}
+
+func (s *StoreBlock) Configure(data interface{}) error {
+	var err error
+
+	err = mapstructure.Decode(data, &s.store)
+	if err != nil {
+		return err
+	}
+	fmt.Printf("%#+v\n", s)
+	return nil
+}
+
+func (s *StoreBlock) Open() error {
+	return nil
+}
+
+func (s *StoreBlock) Close() error {
+	return nil
+}
+
+func (o *ObjectBlock) Allocate(mode uint64, offset uint64, length uint64) error {
+	return unix.Fallocate(int(o.fp.Fd()), uint32(mode), int64(offset), int64(length))
+}
+
+func (o *ObjectBlock) Delete(flags uint64) error {
+	return nil
+	//	return os.Remove(filepath.Join())
+}
+
+func (o *ObjectBlock) ReadAt(data []byte, offset uint64, flags uint64) (uint64, error) {
+	return 0, nil
+}
+
+func (o *ObjectBlock) WriteAt(data []byte, offset uint64, flags uint64) (uint64, error) {
+	return 0, nil
+}
+
+func (o *ObjectBlock) Read(data []byte) (int, error) {
+	return o.fp.Read(data)
+}
+
+func (o *ObjectBlock) Write(data []byte) (int, error) {
+	return o.fp.Write(data)
+}
+
+func (o *ObjectBlock) Seek(offset int64, whence int) (int64, error) {
+	return o.fp.Seek(offset, whence)
+}
+
+func (o *StoreBlock) ObjectExists(name string) (bool, error) {
+	_, err := os.Stat(name)
+	return os.IsNotExist(err), nil
+}
+
+func (o *ObjectBlock) Sync() error {
+	return o.fp.Sync()
+}
+
+func (o *ObjectBlock) Close() error {
+	return o.fp.Close()
+}
+
+func (o *ObjectBlock) Remove() error {
+	return nil
+}
+*/

backend/filesystem/filesystem.go

@@ -0,0 +1,348 @@
+package filesystem
+
+import (
+	"fmt"
+	"hash"
+	"io"
+	"os"
+	"path/filepath"
+	"sync"
+
+	"github.com/mitchellh/mapstructure"
+	"github.com/valyala/fastrand"
+	"golang.org/x/sys/unix"
+	"github.com/sdstack/storage/backend"
+	"github.com/sdstack/storage/cache"
+	shash "github.com/sdstack/storage/hash"
+	"github.com/sdstack/storage/ring"
+)
+
+type BackendFilesystem struct {
+	cfg     *config
+	hash    hash.Hash
+	fdcache cache.Cache
+	rng     fastrand.RNG
+	mu      sync.Mutex
+}
+
+const (
+	vSize = 16 * 1024 * 1024
+)
+
+type config struct {
+	Debug   bool
+	FDCache struct {
+		Engine string
+		Size   int
+	} `mapstructure:"fdcache"`
+	Mode    string `mapstructure:"mode"`
+	Options map[string]interface{}
+	Shards  struct {
+		Data   int
+		Parity int
+	}
+	Ring  string
+	Store []struct {
+		ID     string
+		Path   string
+		Weight int
+	}
+}
+
+func init() {
+	s := &BackendFilesystem{}
+	s.weights = make(map[string]int)
+	backend.RegisterBackend("filesystem", s)
+}
+
+func (s *BackendFilesystem) Init(data interface{}) error {
+	var err error
+	var fi os.FileInfo
+	var statfs unix.Statfs_t
+
+	err = mapstructure.Decode(data, &s.cfg)
+	if err != nil {
+		return err
+	}
+
+	for _, it := range s.cfg.Store {
+		if it.Weight < 0 {
+			continue
+		}
+
+		if fi, err = os.Stat(it.Path); err != nil || !fi.IsDir() {
+			continue
+		}
+
+		if it.Weight > 0 {
+			s.weights[it.Path] = it.Weight
+			continue
+		}
+
+		if err = unix.Statfs(it.Path, &statfs); err == nil {
+			s.weights[it.Path] = int(statfs.Blocks) * int(statfs.Bsize) / vSize
+		}
+	}
+
+	if s.cfg.Mode == "copy" && len(s.weights) < s.cfg.Shards.Data {
+		return fmt.Errorf("data shards is more then available disks")
+	}
+	/*
+		if s.fdcache, err = cache.New(s.cfg.FDCache.Engine, s.cfg.FDCache.Size); err != nil {
+			return err
+		}
+
+		if err = s.fdcache.OnEvict(s.onCacheEvict); err != nil {
+			return err
+		}
+	*/
+	if s.ring, err = ring.New(s.cfg.Ring, s.weights); err != nil {
+		return err
+	}
+
+	if s.hash, err = shash.New("xxhash"); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+/*
+func (s *StoreFilesystem) onCacheEvict(k interface{}, v interface{}) {
+	v.(*os.File).Close()
+}
+*/
+
+func (s *BackendFilesystem) Configure(data interface{}) error {
+	var err error
+	var fi os.FileInfo
+
+	err = mapstructure.Decode(data, &s.cfg)
+	if err != nil {
+		return err
+	}
+
+	for _, it := range s.cfg.Store {
+		if fi, err = os.Stat(it.Path); err != nil || !fi.IsDir() {
+			continue
+		}
+		s.weights[it.Path] = it.Weight
+	}
+
+	if s.cfg.Mode == "copy" && len(s.weights) < s.cfg.Shards.Data {
+		return fmt.Errorf("data shards is more then available disks")
+	}
+
+	s.ring, err = ring.New(s.cfg.Ring, s.weights)
+
+	s.fdcache.Purge()
+	return err
+}
+
+func (s *BackendFilesystem) Allocate(name string, size int64, ndata int, nparity int) error {
+	var err error
+	var fp *os.File
+	var items interface{}
+	var disks []string
+
+	if s.cfg.Debug {
+		fmt.Printf("%T %s\n", s, "allocate")
+	}
+
+	if nparity == 0 && ndata > 0 {
+		if items, err = s.ring.GetItem(name, ndata); err != nil {
+			return err
+		}
+	}
+
+	disks = items.([]string)
+	if s.cfg.Debug {
+		fmt.Printf("%T %s %v %s\n", s, "read", disks, name)
+	}
+
+	var errs []error
+	for _, disk := range disks {
+		if fp, err = os.OpenFile(filepath.Join(disk, name), os.O_RDWR|os.O_CREATE, os.FileMode(0660)); err != nil {
+			continue
+			// TODO: remove from ring
+		}
+		if err = unix.Fallocate(int(fp.Fd()), 0, 0, int64(size)); err != nil {
+			// TODO: remove from ring
+			fp.Close()
+		}
+	}
+
+	if len(errs) > 0 {
+		return errs[0]
+	}
+
+	return nil
+}
+
+func (s *BackendFilesystem) ReadAt(name string, buf []byte, offset int64, ndata int, nparity int) (int, error) {
+	if s.cfg.Debug {
+		fmt.Printf("%T %s\n", s, "read")
+	}
+	//var errs []error
+	var err error
+	var n int
+	var items interface{}
+	var disks []string
+	var fp *os.File
+
+	if nparity == 0 && ndata > 0 {
+		if items, err = s.ring.GetItem(name, ndata); err != nil {
+			return 0, err
+		}
+	}
+
+	disks = items.([]string)
+	if s.cfg.Debug {
+		fmt.Printf("%T %s %v %s %d %d\n", s, "read", disks, name, offset, len(buf))
+	}
+
+	for len(disks) > 0 {
+		s.mu.Lock()
+		disk := disks[int(s.rng.Uint32n(uint32(len(disks))))]
+		s.mu.Unlock()
+		fname := filepath.Join(disk, name)
+		if fp, err = os.OpenFile(fname, os.O_CREATE|os.O_RDWR, os.FileMode(0660)); err != nil {
+			continue
+		}
+		n, err = fp.ReadAt(buf, int64(offset))
+		fp.Close()
+		if err == nil || err == io.EOF {
+			//fmt.Printf("ret from read %d %s\n", n, buf)
+			return n, nil
+		}
+		//		fmt.Printf("aaaa %v\n", err)
+		/*
+			o.fs.ring.DelItem(disk)
+			o.fs.ring.SetState(ring.StateDegrade)
+
+			copy(disks[rnd:], disks[rnd+1:])
+			//		disks[len(disks)-1] = nil
+			disks = disks[:len(disks)-1]
+		*/
+	}
+
+	return 0, backend.ErrIO
+}
+
+type rwres struct {
+	n   int
+	err error
+}
+
+func (s *BackendFilesystem) WriterTo(name string, w io.Writer, offset int64, size int64, ndata int, nparity int) (int64, error) {
+
+	buf := make([]byte, size)
+	n, err := s.ReadAt(name, buf, offset, ndata, nparity)
+	if err != nil || int64(n) < size {
+		return 0, backend.ErrIO
+	}
+	n, err = w.Write(buf)
+	return int64(n), err
+}
+
+func (s *BackendFilesystem) ReaderFrom(name string, r io.Reader, offset int64, size int64, ndata int, nparity int) (int64, error) {
+	buf := make([]byte, size)
+	n, err := r.Read(buf)
+	if err != nil || int64(n) < size {
+		return 0, backend.ErrIO
+	}
+	n, err = s.WriteAt(name, buf, offset, ndata, nparity)
+	return int64(n), err
+}
+
+func (s *BackendFilesystem) WriteAt(name string, buf []byte, offset int64, ndata int, nparity int) (int, error) {
+	if s.cfg.Debug {
+		fmt.Printf("%T %s\n", s, "write")
+	}
+
+	var n int
+	var err error
+	var items interface{}
+	var disks []string
+	var fp *os.File
+	if nparity == 0 && ndata > 0 {
+		if items, err = s.ring.GetItem(name, ndata); err != nil {
+			return 0, err
+		}
+	}
+	disks = items.([]string)
+	hinfo := struct {
+		Hashes []uint64
+	}{}
+	_ = hinfo
+
+	//result := make(chan rwres, len(disks))
+	for _, disk := range disks {
+		//		go func() {
+		//var res rwres
+
+		fp, err = os.OpenFile(filepath.Join(disk, name), os.O_CREATE|os.O_RDWR, os.FileMode(0660))
+		if err != nil {
+			continue
+		}
+
+		//mw := io.MultiWriter{s.hash, fp
+		n, err = fp.WriteAt(buf, int64(offset))
+		/*
+			if o.fs.cfg.Options.Sync {
+				if res.err = fp.Sync(); res.err != nil {
+					result <- res
+				}
+			}
+		*/
+		fp.Close()
+	}
+
+	if s.ring.Size() < 1 {
+		s.ring.SetState(ring.StateFail)
+		return 0, fmt.Errorf("can't write to failed ring")
+	}
+	/*
+		if res.err != nil || res.n < len(buf) {
+			n = res.n
+			err = res.err
+			//delete(o.fps, res.disk)
+		}
+	*/
+	return n, nil
+}
+
+func (s *BackendFilesystem) Exists(name string, ndata int, nparity int) (bool, error) {
+	if s.cfg.Debug {
+		fmt.Printf("%T %s %s\n", s, "object_exists", name)
+	}
+
+	var err error
+	var items interface{}
+	var disks []string
+	if nparity == 0 && ndata > 0 {
+		if items, err = s.ring.GetItem(name, ndata); err != nil {
+			return false, err
+		}
+	}
+	disks = items.([]string)
+
+	for len(disks) > 0 {
+		s.mu.Lock()
+		disk := disks[int(s.rng.Uint32n(uint32(len(disks))))]
+		s.mu.Unlock()
+		if _, err = os.Stat(filepath.Join(disk, name)); err == nil {
+			break
+		}
+	}
+
+	return !os.IsNotExist(err), nil
+}
+
+func (s *BackendFilesystem) Remove(name string, ndata int, nparity int) error {
+	if s.cfg.Debug {
+		fmt.Printf("%T %s\n", s, "remove")
+	}
+
+	return nil
+}

bench/hash/bench_test.go

@@ -0,0 +1,237 @@
+package main
+
+import (
+	"crypto/rand"
+	"crypto/sha1"
+	"crypto/sha256"
+	"hash/crc32"
+	"testing"
+
+	creachadairCity "bitbucket.org/creachadair/cityhash"
+	jpathyCity "bitbucket.org/jpathy/dmc/cityhash"
+	"github.com/OneOfOne/xxhash"
+	dgryskiSpooky "github.com/dgryski/go-spooky"
+	huichenMurmur "github.com/huichen/murmur"
+	farmhash "github.com/leemcloughlin/gofarmhash"
+	"github.com/minio/blake2b-simd"
+	"github.com/minio/highwayhash"
+	sha256simd "github.com/minio/sha256-simd"
+	reuseeMurmur "github.com/reusee/mmh3"
+	hashlandSpooky "github.com/tildeleb/hashland/spooky"
+	zhangMurmur "github.com/zhangxinngang/murmur"
+)
+
+var result interface{}
+
+func mkinput(n int) []byte {
+	rv := make([]byte, n)
+	rand.Read(rv)
+	return rv
+}
+
+func benchmarkHash(num int, fn func(i int)) {
+	for n := 0; n < num; n++ {
+		fn(n)
+	}
+}
+
+const benchSize = 32 << 20
+
+func BenchmarkHighwayhash64(b *testing.B) {
+	input := mkinput(benchSize)
+	b.SetBytes(int64(len(input)))
+	b.ResetTimer()
+	key := make([]byte, 32)
+	_, err := rand.Read(key)
+	if err != nil {
+		panic(err)
+	}
+	h, err := highwayhash.New64(key)
+	if err != nil {
+		panic(err)
+	}
+
+	benchmarkHash(b.N, func(n int) {
+		_ = h.Sum(input)
+	})
+}
+
+func BenchmarkFarmHashHash32(b *testing.B) {
+	input := mkinput(benchSize)
+	b.SetBytes(int64(len(input)))
+	b.ResetTimer()
+	benchmarkHash(b.N, func(n int) {
+		_ = farmhash.Hash32(input)
+	})
+}
+func BenchmarkFarmHashHash64(b *testing.B) {
+	input := mkinput(benchSize)
+	b.SetBytes(int64(len(input)))
+	b.ResetTimer()
+	benchmarkHash(b.N, func(n int) {
+		_ = farmhash.Hash64(input)
+	})
+}
+func BenchmarkHuichenMurmur(b *testing.B) {
+	input := mkinput(benchSize)
+	b.SetBytes(int64(len(input)))
+	b.ResetTimer()
+	benchmarkHash(b.N, func(n int) {
+		_ = huichenMurmur.Murmur3(input)
+	})
+}
+func BenchmarkReuseeMurmur(b *testing.B) {
+	input := mkinput(benchSize)
+	b.SetBytes(int64(len(input)))
+	b.ResetTimer()
+	benchmarkHash(b.N, func(n int) {
+		_ = reuseeMurmur.Sum32(input)
+	})
+}
+func BenchmarkZhangMurmur(b *testing.B) {
+	input := mkinput(benchSize)
+	b.SetBytes(int64(len(input)))
+	b.ResetTimer()
+	benchmarkHash(b.N, func(n int) {
+		_ = zhangMurmur.Murmur3(input)
+	})
+}
+func BenchmarkDgryskiSpooky32(b *testing.B) {
+	input := mkinput(benchSize)
+	b.SetBytes(int64(len(input)))
+	b.ResetTimer()
+	benchmarkHash(b.N, func(n int) {
+		_ = dgryskiSpooky.Hash32(input)
+	})
+}
+func BenchmarkDgryskiSpooky64(b *testing.B) {
+	input := mkinput(benchSize)
+	b.SetBytes(int64(len(input)))
+	b.ResetTimer()
+	benchmarkHash(b.N, func(n int) {
+		_ = dgryskiSpooky.Hash64(input)
+	})
+}
+func BenchmarkHashlandSpooky32(b *testing.B) {
+	input := mkinput(benchSize)
+	b.SetBytes(int64(len(input)))
+	b.ResetTimer()
+	benchmarkHash(b.N, func(n int) {
+		_ = hashlandSpooky.Hash32(input, 0)
+	})
+}
+func BenchmarkHashlandSpooky64(b *testing.B) {
+	input := mkinput(benchSize)
+	b.SetBytes(int64(len(input)))
+	b.ResetTimer()
+	benchmarkHash(b.N, func(n int) {
+		_ = hashlandSpooky.Hash64(input, 0)
+	})
+}
+func BenchmarkHashlandSpooky128(b *testing.B) {
+	input := mkinput(benchSize)
+	b.SetBytes(int64(len(input)))
+	b.ResetTimer()
+	benchmarkHash(b.N, func(n int) {
+		_, _ = hashlandSpooky.Hash128(input, 0)
+	})
+}
+func BenchmarkJPathyCity32(b *testing.B) {
+	input := mkinput(benchSize)
+	b.SetBytes(int64(len(input)))
+	b.ResetTimer()
+	benchmarkHash(b.N, func(n int) {
+		_ = jpathyCity.Hash32(input)
+	})
+}
+func BenchmarkCreachadairCity32(b *testing.B) {
+	input := mkinput(benchSize)
+	b.SetBytes(int64(len(input)))
+	b.ResetTimer()
+	benchmarkHash(b.N, func(n int) {
+		_ = creachadairCity.Hash32(input)
+	})
+}
+func BenchmarkCreachadairCity64(b *testing.B) {
+	input := mkinput(benchSize)
+	b.SetBytes(int64(len(input)))
+	b.ResetTimer()
+	benchmarkHash(b.N, func(n int) {
+		_ = creachadairCity.Hash64(input)
+	})
+}
+func BenchmarkCreachadairCity128(b *testing.B) {
+	input := mkinput(benchSize)
+	b.SetBytes(int64(len(input)))
+	b.ResetTimer()
+	benchmarkHash(b.N, func(n int) {
+		_, _ = creachadairCity.Hash128(input)
+	})
+}
+
+func BenchmarkXxHash32(b *testing.B) {
+	input := mkinput(benchSize)
+	b.SetBytes(int64(len(input)))
+	b.ResetTimer()
+	benchmarkHash(b.N, func(n int) {
+		_ = xxhash.Checksum32(input)
+	})
+}
+
+func BenchmarkXxHash64(b *testing.B) {
+	input := mkinput(benchSize)
+	b.SetBytes(int64(len(input)))
+	b.ResetTimer()
+	benchmarkHash(b.N, func(n int) {
+		_ = xxhash.Checksum64(input)
+	})
+}
+
+func BenchmarkBlake2bSIMD256(b *testing.B) {
+	input := mkinput(benchSize)
+	h := blake2b.New256()
+	b.SetBytes(int64(len(input)))
+	b.ResetTimer()
+	benchmarkHash(b.N, func(n int) {
+		h.Reset()
+		_, _ = h.Write(input)
+		h.Sum(nil)
+	})
+}
+
+func BenchmarkSHA256(b *testing.B) {
+	input := mkinput(benchSize)
+	b.SetBytes(int64(len(input)))
+	b.ResetTimer()
+	benchmarkHash(b.N, func(n int) {
+		_ = sha256.Sum256(input)
+	})
+}
+func BenchmarkSHA1(b *testing.B) {
+	input := mkinput(benchSize)
+	b.SetBytes(int64(len(input)))
+	b.ResetTimer()
+	benchmarkHash(b.N, func(n int) {
+		_ = sha1.Sum(input)
+	})
+}
+func BenchmarkCRC32(b *testing.B) {
+	input := mkinput(benchSize)
+	b.SetBytes(int64(len(input)))
+	b.ResetTimer()
+	h := crc32.NewIEEE()
+	benchmarkHash(b.N, func(n int) {
+		_ = h.Sum(input)
+	})
+}
+func BenchmarkSHA256SIMBD(b *testing.B) {
+	input := mkinput(benchSize)
+	b.SetBytes(int64(len(input)))
+	b.ResetTimer()
+	benchmarkHash(b.N, func(n int) {
+		_ = sha256simd.Sum256(input)
+	})
+}
+func main() {
+
+}

bench/rand/bench_test.go

@@ -0,0 +1,42 @@
+package main
+
+import (
+	"math/rand"
+	"sync"
+	"sync/atomic"
+	"testing"
+	"time"
+
+	"github.com/valyala/fastrand"
+)
+
+var BenchSink uint32
+
+func BenchmarkMathRandIntn(b *testing.B) {
+	rand.Seed(time.Now().Unix())
+	b.RunParallel(func(pb *testing.PB) {
+		s := uint32(0)
+		for pb.Next() {
+			s += uint32(rand.Intn(1e6))
+		}
+		atomic.AddUint32(&BenchSink, s)
+	})
+}
+
+func BenchmarkRNGUint32nWithLock(b *testing.B) {
+	var r fastrand.RNG
+	var rMu sync.Mutex
+	b.RunParallel(func(pb *testing.PB) {
+		s := uint32(0)
+		for pb.Next() {
+			rMu.Lock()
+			s += r.Uint32n(1e6)
+			rMu.Unlock()
+		}
+		atomic.AddUint32(&BenchSink, s)
+	})
+}
+
+func main() {
+
+}

cache/block/block.go

@@ -0,0 +1 @@
+package block

cache/cache.go

@@ -0,0 +1,98 @@
+package cache
+
+import (
+	"fmt"
+	"strings"
+)
+
+type Cache interface {
+	Configure(interface{}) error
+	Set(interface{}, interface{}) error
+	Exists(interface{}) (bool, error)
+	Get(interface{}) (interface{}, error)
+	Keys() ([]interface{}, error)
+	//Size() int
+	Purge() error
+	Del(interface{}) error
+	OnEvict(func(interface{}, interface{})) error
+}
+
+var cacheTypes map[string]Cache
+
+func init() {
+	cacheTypes = make(map[string]Cache)
+	RegisterCache("", &Dummy{})
+}
+
+func RegisterCache(engine string, cache Cache) {
+	cacheTypes[engine] = cache
+}
+
+func New(ctype string, cfg interface{}) (Cache, error) {
+	var err error
+
+	cache, ok := cacheTypes[ctype]
+	if !ok {
+		return nil, fmt.Errorf("unknown cluster type %s. only %s supported", ctype, strings.Join(CacheTypes(), ","))
+	}
+
+	if cfg == nil {
+		return cache, nil
+	}
+
+	err = cache.Configure(cfg)
+	if err != nil {
+		return nil, err
+	}
+
+	return cache, nil
+}
+
+func CacheTypes() []string {
+	var ctypes []string
+	for ctype, _ := range cacheTypes {
+		ctypes = append(ctypes, ctype)
+	}
+	return ctypes
+}
+
+type Dummy struct{}
+
+func (*Dummy) Configure(interface{}) error {
+	return nil
+}
+
+func (*Dummy) Del(interface{}) error {
+	return nil
+}
+
+func (*Dummy) Set(interface{}, interface{}) error {
+	return nil
+}
+func (*Dummy) Get(interface{}) (interface{}, error) {
+	return nil, nil
+}
+
+func (*Dummy) Exists(interface{}) (bool, error) {
+	return false, nil
+}
+
+func (*Dummy) Keys() ([]interface{}, error) {
+	return nil, nil
+}
+
+func (*Dummy) OnEvict(func(interface{}, interface{})) error {
+	return nil
+}
+
+func (*Dummy) Peek(interface{}) (interface{}, error) {
+	return nil, nil
+}
+
+func (*Dummy) Purge() error {
+	return nil
+}
+
+func (*Dummy) Size() int {
+	return 0
+}

cache/filesystem/filesystem.go

@@ -0,0 +1 @@
+package filesystem

cache/memory/memory.go

@@ -0,0 +1,81 @@
+package memory
+
+import (
+	"errors"
+
+	hcache "github.com/hashicorp/golang-lru"
+	"github.com/sdstack/storage/cache"
+)
+
+type config struct {
+	Size int
+}
+
+type CacheLRU struct {
+	cfg *config
+	c   *hcache.Cache
+}
+
+func init() {
+	cache.RegisterCache("memory-lru", &CacheLRU{})
+}
+
+func (c *CacheLRU) Configure(data interface{}) error {
+	//	var err error
+	/*
+		err = mapstructure.Decode(data, &c.cfg)
+		if err != nil {
+			return err
+		}
+	*/
+	c.cfg = &config{Size: data.(int)}
+	return nil
+}
+
+func (c *CacheLRU) OnEvict(onEvict func(interface{}, interface{})) error {
+	var err error
+	c.c, err = hcache.NewWithEvict(c.cfg.Size, onEvict)
+	return err
+}
+
+func (c *CacheLRU) Peek(k interface{}) (interface{}, bool) {
+	return c.c.Peek(k)
+}
+
+func (c *CacheLRU) Purge() error {
+	c.c.Purge()
+	return nil
+}
+
+func (c *CacheLRU) Size() int {
+	return c.c.Len()
+}
+
+func (c *CacheLRU) Set(k interface{}, v interface{}) error {
+	if c.c.Add(k, v) {
+		return nil
+	}
+	return errors.New("failed")
+}
+
+func (c *CacheLRU) Del(k interface{}) error {
+	c.c.Remove(k)
+	return nil
+}
+
+func (c *CacheLRU) Get(k interface{}) (interface{}, error) {
+	v, ok := c.c.Get(k)
+	if ok {
+		return v, nil
+	}
+	return v, errors.New("failed")
+}
+
+func (c *CacheLRU) Keys() ([]interface{}, error) {
+	v := c.c.Keys()
+	return v, nil
+}
+
+func (c *CacheLRU) Exists(k interface{}) (bool, error) {
+	return c.c.Contains(k), nil
+}

cluster/cluster.go

@@ -0,0 +1,87 @@
+package cluster
+
+import (
+	"fmt"
+	"strings"
+)
+
+var clusterTypes map[string]Cluster
+
+func init() {
+	clusterTypes = make(map[string]Cluster)
+	RegisterCluster("none", &clusterNone{})
+}
+func RegisterCluster(engine string, cluster Cluster) {
+	clusterTypes[engine] = cluster
+}
+
+// Info struct contains cluster info
+type Info struct {
+	BlockSize uint32
+	Mode      string
+}
+
+// Member struct contains info about cluster member
+type Member struct {
+	UUID    []byte
+	Name    string
+	Network string
+	Host    string
+	Port    string
+}
+
+// Cluster represents cluster interface
+type Cluster interface {
+	Start() error
+	Stop() error
+	Configure(interface{}) error
+	//	Format() error
+	//	Check() error
+	//	Recover() error
+	//	Info() (*Info, error)
+	//	Snapshot() error
+	//	Reweight() error
+	//	Members() []Member
+}
+
+func New(ctype string, cfg interface{}) (Cluster, error) {
+	var err error
+
+	cluster, ok := clusterTypes[ctype]
+	if !ok {
+		return nil, fmt.Errorf("unknown cluster type %s. only %s supported", ctype, strings.Join(ClusterTypes(), ","))
+	}
+
+	if cfg == nil {
+		return cluster, nil
+	}
+
+	err = cluster.Configure(cfg)
+	if err != nil {
+		return nil, err
+	}
+
+	return cluster, nil
+}
+
+func ClusterTypes() []string {
+	var ctypes []string
+	for ctype, _ := range clusterTypes {
+		ctypes = append(ctypes, ctype)
+	}
+	return ctypes
+}
+
+type clusterNone struct{}
+
+func (c *clusterNone) Configure(data interface{}) error {
+	return nil
+}
+
+func (c *clusterNone) Start() error {
+	return nil
+}
+
+func (c *clusterNone) Stop() error {
+	return nil
+}

cluster/etcdint/etcdint.go

@@ -0,0 +1,65 @@
+package etcdint
+
+import (
+	"errors"
+	"time"
+
+	"github.com/sdstack/storage/cluster"
+
+	"github.com/coreos/etcd/embed"
+	"github.com/coreos/etcd/wal"
+	"github.com/mitchellh/mapstructure"
+)
+
+type config struct {
+	WalSize int64  `mapstructure:"wal_size"`
+	Store   string `mapstructure:"store"`
+}
+
+// Internal strect holds data used by internal cluster engine
+type ClusterEtcdint struct {
+	etcdsrv *embed.Etcd
+	etcdcfg *embed.Config
+	cfg     *config
+}
+
+func init() {
+	cluster.RegisterCluster("etcdint", &ClusterEtcdint{})
+}
+
+func (c *ClusterEtcdint) Configure(data interface{}) error {
+	var err error
+
+	c.etcdcfg = embed.NewConfig()
+	err = mapstructure.Decode(data, &c.cfg)
+	if err != nil {
+		return err
+	}
+	c.etcdcfg.Dir = c.cfg.Store
+
+	return nil
+}
+
+// Start internal cluster engine
+func (c *ClusterEtcdint) Start() error {
+	wal.SegmentSizeBytes = c.cfg.WalSize
+	e, err := embed.StartEtcd(c.etcdcfg)
+	if err != nil {
+		return err
+	}
+	select {
+	case <-e.Server.ReadyNotify():
+		c.etcdsrv = e
+		break
+	case <-time.After(60 * time.Second):
+		e.Server.Stop() // trigger a shutdown
+		return errors.New("Server took too long to start!")
+	}
+	return nil
+}
+
+// Stop internal cluster engin
+func (c *ClusterEtcdint) Stop() error {
+	c.etcdsrv.Server.Stop()
+	return nil
+}

hash/hash.go

@@ -0,0 +1,34 @@
+package hash
+
+import (
+	"fmt"
+	"hash"
+	"strings"
+)
+
+var hashTypes map[string]hash.Hash
+
+func init() {
+	hashTypes = make(map[string]hash.Hash)
+}
+
+func RegisterHash(engine string, hash hash.Hash) {
+	hashTypes[engine] = hash
+}
+
+func New(htype string) (hash.Hash, error) {
+	hash, ok := hashTypes[htype]
+	if !ok {
+		return nil, fmt.Errorf("unknown hash type %s. only %s supported", htype, strings.Join(HashTypes(), ","))
+	}
+
+	return hash, nil
+}
+
+func HashTypes() []string {
+	var htypes []string
+	for htype, _ := range hashTypes {
+		htypes = append(htypes, htype)
+	}
+	return htypes
+}

hash/xxhash/xxhash.go

@@ -0,0 +1,11 @@
+package xxhash
+
+import (
+	"github.com/OneOfOne/xxhash"
+
+	"github.com/sdstack/storage/hash"
+)
+
+func init() {
+	hash.RegisterHash("xxhash", &xxhash.XXHash64{})
+}

journal/journal.go

@@ -0,0 +1,91 @@
+package journal
+
+import (
+	"fmt"
+	"strings"
+)
+
+type Journal interface {
+	Write([]byte, []byte) (int, error)
+	Read([]byte, []byte) (int, error)
+	Configure(interface{}) error
+}
+
+var journalTypes map[string]Journal
+
+func init() {
+	journalTypes = make(map[string]Journal)
+}
+
+func RegisterJournal(engine string, journal Journal) {
+	journalTypes[engine] = journal
+}
+
+func New(ctype string, cfg interface{}) (Journal, error) {
+	var err error
+
+	journal, ok := journalTypes[ctype]
+	if !ok {
+		return nil, fmt.Errorf("unknown cluster type %s. only %s supported", ctype, strings.Join(JournalTypes(), ","))
+	}
+
+	if cfg == nil {
+		return journal, nil
+	}
+
+	err = journal.Configure(cfg)
+	if err != nil {
+		return nil, err
+	}
+
+	return journal, nil
+}
+
+func JournalTypes() []string {
+	var ctypes []string
+	for ctype, _ := range journalTypes {
+		ctypes = append(ctypes, ctype)
+	}
+	return ctypes
+}
+
+type Dummy struct{}
+
+func (*Dummy) Configure(interface{}) error {
+	return nil
+}
+
+func (*Dummy) Del(interface{}) error {
+	return nil
+}
+
+func (*Dummy) Set(interface{}, interface{}) error {
+	return nil
+}
+func (*Dummy) Get(interface{}) (interface{}, error) {
+	return nil, nil
+}
+
+func (*Dummy) Exists(interface{}) (bool, error) {
+	return false, nil
+}
+
+func (*Dummy) Keys() ([]interface{}, error) {
+	return nil, nil
+}
+
+func (*Dummy) OnEvict(func(interface{}, interface{})) error {
+	return nil
+}
+
+func (*Dummy) Peek(interface{}) (interface{}, error) {
+	return nil, nil
+}
+
+func (*Dummy) Purge() error {
+	return nil
+}
+
+func (*Dummy) Size() int {
+	return 0
+}

journal/leveldb/leveldb.go

@@ -0,0 +1,76 @@
+package leveldb
+
+import (
+	"github.com/syndtr/goleveldb/leveldb"
+	"github.com/syndtr/goleveldb/leveldb/opt"
+	"github.com/sdstack/storage/cache"
+)
+
+type config struct {
+	Store string
+}
+
+type LevelDB struct {
+	db *leveldb.DB
+}
+
+func init() {
+	cache.RegisterCache("leveldb", &LevelDB{})
+}
+
+func (c *LevelDB) Configure(cfg interface{}) error {
+	return nil
+}
+
+func (c *LevelDB) Open() error {
+	var err error
+	c.db, err = leveldb.OpenFile("/srv/store/sc01/db", &opt.Options{
+		Compression: opt.NoCompression,
+	})
+	if err != nil {
+		return err
+	}
+	return nil
+}
+
+func (c *LevelDB) Close() error {
+	return c.db.Close()
+}
+
+func (c *LevelDB) Set(k interface{}, v interface{}) error {
+	return c.db.Put(k.([]byte), v.([]byte), nil)
+}
+
+func (c *LevelDB) Get(k interface{}) (interface{}, error) {
+	return c.db.Get(k.([]byte), nil)
+}
+
+func (c *LevelDB) Del(k interface{}) error {
+	return c.db.Delete(k.([]byte), nil)
+}
+
+func (c *LevelDB) Exists(k interface{}) (bool, error) {
+	return c.db.Has(k.([]byte), nil)
+}
+
+func (c *LevelDB) Keys() ([]interface{}, error) {
+	var keys []interface{}
+	var err error
+
+	iter := c.db.NewIterator(nil, nil)
+	for iter.Next() {
+		keys = append(keys, string(iter.Key()))
+	}
+
+	iter.Release()
+	err = iter.Error()
+	return keys, err
+}
+
+func (c *LevelDB) OnEvict(f func(interface{}, interface{})) error {
+	return nil
+}
+
+func (c *LevelDB) Purge() error {
+	return nil
+}

kv/storage.go

@@ -0,0 +1,119 @@
+package kv
+
+import (
+	"fmt"
+	"io"
+
+	"github.com/sdstack/storage/backend"
+	"github.com/sdstack/storage/cache"
+	"github.com/sdstack/storage/cluster"
+)
+
+type KV struct {
+	backend backend.Backend
+	cluster cluster.Cluster
+	cache   cache.Cache
+}
+
+type Cluster struct {
+	Engine string
+	Config interface{}
+}
+
+type Store struct {
+	Engine string
+	Config interface{}
+}
+
+func New(cfg interface{} /* c cluster.Cluster, b backend.Backend */) (*KV, error) {
+	var err error
+
+	/*
+		clusterEngine := viper.GetStringMap("cluster")["engine"].(string)
+		cluster, err := cluster.New(clusterEngine, viper.GetStringMap("cluster")[clusterEngine])
+		if err != nil {
+			return nil, err
+		}
+
+		if err = cluster.Start(); err != nil {
+			log.Printf("cluster start error %s", err)
+			os.Exit(1)
+		}
+		defer cluster.Stop()
+
+		return &KV{backend: b, cluster: c}, nil
+	*/
+	return &KV{}, err
+}
+
+func (e *KV) SetBackend(b backend.Backend) error {
+	if e.backend != nil {
+		return fmt.Errorf("backend already set")
+	}
+	e.backend = b
+
+	return nil
+}
+
+func (e *KV) SetCluster(c cluster.Cluster) error {
+	if e.cluster != nil {
+		return fmt.Errorf("cluster already set")
+	}
+	e.cluster = c
+
+	return nil
+}
+
+func (e *KV) SetCache(c cache.Cache) {
+	e.cache = c
+}
+
+func (e *KV) Exists(s string, ndata int, nparity int) (bool, error) {
+	return e.backend.Exists(s, ndata, nparity)
+}
+
+func (e *KV) Allocate(name string, size int64, ndata int, nparity int) error {
+	return e.backend.Allocate(name, size, ndata, nparity)
+}
+
+func (e *KV) Remove(name string, ndata int, nparity int) error {
+	return e.backend.Remove(name, ndata, nparity)
+}
+
+func (e *KV) WriteAt(name string, buf []byte, offset int64, ndata int, nparity int) (int, error) {
+	return e.backend.WriteAt(name, buf, offset, ndata, nparity)
+}
+
+func (e *KV) ReadAt(name string, buf []byte, offset int64, ndata int, nparity int) (int, error) {
+	return e.backend.ReadAt(name, buf, offset, ndata, nparity)
+}
+
+type RW struct {
+	Name    string
+	KV      *KV
+	Offset  int64
+	Size    int64
+	Ndata   int
+	Nparity int
+}
+
+func (e *RW) Seek(offset int64, whence int) (int64, error) {
+	e.Offset = offset
+	return offset, nil
+}
+
+func (e *RW) Read(buf []byte) (int, error) {
+	return e.KV.backend.ReadAt(e.Name, buf, e.Offset, e.Ndata, e.Nparity)
+}
+
+func (e *RW) ReaderFrom(r io.Reader) (int64, error) {
+	return e.KV.backend.ReaderFrom(e.Name, r, e.Offset, e.Size, e.Ndata, e.Nparity)
+}
+
+func (e *RW) Write(buf []byte) (int, error) {
+	return e.KV.backend.WriteAt(e.Name, buf, e.Offset, e.Ndata, e.Nparity)
+}
+
+func (e *RW) WriterTo(w io.Writer) (int64, error) {
+	return e.KV.backend.WriterTo(e.Name, w, e.Offset, e.Size, e.Ndata, e.Nparity)
+}

proxy/proxy.go

@@ -0,0 +1,56 @@
+package proxy
+
+import (
+	"fmt"
+	"strings"
+
+	"github.com/sdstack/storage/kv"
+)
+
+var proxyTypes map[string]Proxy
+
+func init() {
+	proxyTypes = make(map[string]Proxy)
+}
+
+func RegisterProxy(engine string, proxy Proxy) {
+	proxyTypes[engine] = proxy
+}
+
+// Proxy represents proxy interface
+type Proxy interface {
+	Start() error
+	Stop() error
+	Configure(*kv.KV, interface{}) error
+	//	Format() error
+	//	Check() error
+	//	Recover() error
+	//	Info() (*Info, error)
+	//	Snapshot() error
+	//	Reweight() error
+	//	Members() []Member
+}
+
+func New(ptype string, cfg interface{}, engine *kv.KV) (Proxy, error) {
+	var err error
+
+	proxy, ok := proxyTypes[ptype]
+	if !ok {
+		return nil, fmt.Errorf("unknown proxy type %s, only %s supported", ptype, strings.Join(ProxyTypes(), ","))
+	}
+
+	err = proxy.Configure(engine, cfg)
+	if err != nil {
+		return nil, err
+	}
+
+	return proxy, nil
+}
+
+func ProxyTypes() []string {
+	var ptypes []string
+	for ptype, _ := range proxyTypes {
+		ptypes = append(ptypes, ptype)
+	}
+	return ptypes
+}

sds-storage/.gitignore

@@ -0,0 +1,3 @@
+sds-storage
+data/
+test/

sds-storage/Makefile

@@ -0,0 +1,17 @@
+FLAGS_DEFAULT := 'proxy_sheepdog backend_filesystem transport_tcp hash_xxhash'
+FLAGS_MINIMAL := 'proxy_sheepdog backend_filesystem transport_tcp hash_xxhash'
+
+all:
+	go build -tags $(FLAGS_DEFAULT)
+
+release:
+	go build -tags $(FLAGS_DEFAULT) --ldflags '-s -w'
+
+minimal:
+	go build -tags $(FLAGS_MINIMAL)
+
+test:
+	#sudo qemu-nbd -f raw --cache=none --aio=threads --discard=unmap --detect-zeroes=unmap -c /dev/nbd0 sheepdog:test
+	#fio
+	#qemu-img create -f raw sheepdog:127.0.0.1:7000:test 5G
+	#qemu-system-x86_64 -machine q35 -cpu kvm64 -smp 2 -accel kvm -m 512M -vnc 0.0.0.0:10 -device virtio-scsi-pci,id=scsi0,iothread=iothread0 -drive aio=threads,rerror=stop,werror=stop,if=none,format=raw,id=drive-scsi-disk0,cache=none,file=sheepdog:test,discard=unmap,detect-zeroes=off  -device scsi-hd,bus=scsi0.0,drive=drive-scsi-disk0,id=device-scsi-disk0 -object iothread,id=iothread0

sds-storage/backend_block.go

@@ -0,0 +1,7 @@
+// +build backend_block
+
+package main
+
+import (
+	_ "github.com/sdstack/storage/backend/block"
+)

sds-storage/backend_filesystem.go

@@ -0,0 +1,7 @@
+// +build backend_filesystem
+
+package main
+
+import (
+	_ "github.com/sdstack/storage/backend/filesystem"
+)

sds-storage/cache_block.go

@@ -0,0 +1,7 @@
+// +build cache_block
+
+package main
+
+import (
+	_ "github.com/sdstack/storage/cache/block"
+)

sds-storage/cache_filesystem.go

@@ -0,0 +1,7 @@
+// +build cache_filesystem
+
+package main
+
+import (
+	_ "github.com/sdstack/storage/cache/filesystem"
+)

sds-storage/cache_leveldb.go

@@ -0,0 +1,7 @@
+// +build journal_leveldb
+
+package main
+
+import (
+	_ "github.com/sdstack/storage/journal/leveldb"
+)

sds-storage/cache_memory.go

@@ -0,0 +1,7 @@
+// +build cache_memory
+
+package main
+
+import (
+	_ "github.com/sdstack/storage/cache/memory"
+)

sds-storage/cluster_etcdint.go

@@ -0,0 +1,7 @@
+// +build cluster_etcdint
+
+package main
+
+import (
+	_ "github.com/sdstack/storage/cluster/etcdint"
+)

sds-storage/cmd/block.go

@@ -0,0 +1,44 @@
+package cmd
+
+import (
+	"fmt"
+
+	"github.com/spf13/cobra"
+)
+
+// blockCmd represents the node command
+var blockCmd = &cobra.Command{
+	Use:   "block",
+	Short: "Control block devices",
+	Long: `A longer description that spans multiple lines and likely contains examples
+and usage of using your command. For example:
+
+Cobra is a CLI library for Go that empowers applications.
+This application is a tool to generate the needed files
+to quickly create a Cobra application.`,
+	Run: func(cmd *cobra.Command, args []string) {
+		fmt.Println("block called")
+		fmt.Println("Available Commands:")
+		for _, c := range cmd.Commands() {
+			if c.IsAvailableCommand() {
+				fmt.Printf("  %s\t%s\n", c.Name(), c.Short)
+			}
+		}
+	},
+}
+
+func init() {
+	blockCmd.AddCommand(blockSCSICmd)
+	blockCmd.AddCommand(blockNBDCmd)
+	rootCmd.AddCommand(blockCmd)
+
+	// Here you will define your flags and configuration settings.
+
+	// Cobra supports Persistent Flags which will work for this command
+	// and all subcommands, e.g.:
+	// nodeCmd.PersistentFlags().String("foo", "", "A help for foo")
+
+	// Cobra supports local flags which will only run when this command
+	// is called directly, e.g.:
+	// nodeCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle")
+}

sds-storage/cmd/block_nbd.go

@@ -0,0 +1,21 @@
+package cmd
+
+import (
+	"fmt"
+
+	"github.com/spf13/cobra"
+)
+
+var blockNBDCmd = &cobra.Command{
+	Use:   "nbd",
+	Short: "A brief description of your command",
+	Long: `A longer description that spans multiple lines and likely contains examples
+and usage of using your command. For example:
+
+Cobra is a CLI library for Go that empowers applications.
+This application is a tool to generate the needed files
+to quickly create a Cobra application.`,
+	Run: func(cmd *cobra.Command, args []string) {
+		fmt.Println("block nbd called")
+	},
+}

sds-storage/cmd/block_scsi.go

@@ -0,0 +1,32 @@
+package cmd
+
+import (
+	"fmt"
+
+	"github.com/spf13/cobra"
+)
+
+var blockSCSICmd = &cobra.Command{
+	Use:   "scsi",
+	Short: "A brief description of your command",
+	Long: `A longer description that spans multiple lines and likely contains examples
+and usage of using your command. For example:
+
+Cobra is a CLI library for Go that empowers applications.
+This application is a tool to generate the needed files
+to quickly create a Cobra application.`,
+	Run: func(cmd *cobra.Command, args []string) {
+		fmt.Println("block scsi called")
+		fmt.Println("Available Commands:")
+		for _, c := range cmd.Commands() {
+			if c.IsAvailableCommand() {
+				fmt.Printf("  %s\t%s\n", c.Name(), c.Short)
+			}
+		}
+	},
+}
+
+func init() {
+	blockSCSICmd.AddCommand(blockSCSIAttachCmd)
+	//rootCmd.AddCommand(blockCmd)
+}

sds-storage/cmd/block_scsi_attach.go

@@ -0,0 +1,21 @@
+package cmd
+
+import "github.com/spf13/cobra"
+
+var blockSCSIAttachCmd = &cobra.Command{
+	Use:   "attach",
+	Short: "attach volume to scsi device",
+	Long: `A longer description that spans multiple lines and likely contains examples
+and usage of using your command. For example:
+
+Cobra is a CLI library for Go that empowers applications.
+This application is a tool to generate the needed files
+to quickly create a Cobra application.`,
+	Run: func(cmd *cobra.Command, args []string) {
+		blockSCSIAttachAction(cmd, args)
+	},
+}
+
+func blockSCSIAttachAction(cmd *cobra.Command, args []string) error {
+	return nil
+}

sds-storage/cmd/cluster.go

@@ -0,0 +1,44 @@
+package cmd
+
+import (
+	"fmt"
+
+	"github.com/spf13/cobra"
+)
+
+// clusterCmd represents the node command
+var clusterCmd = &cobra.Command{
+	Use:   "cluster",
+	Short: "Control cluster",
+	Long: `A longer description that spans multiple lines and likely contains examples
+and usage of using your command. For example:
+
+Cobra is a CLI library for Go that empowers applications.
+This application is a tool to generate the needed files
+to quickly create a Cobra application.`,
+	Run: func(cmd *cobra.Command, args []string) {
+		fmt.Println("cluster called")
+		fmt.Println("Available Commands:")
+		for _, c := range cmd.Commands() {
+			if c.IsAvailableCommand() {
+				fmt.Printf("  %s\t%s\n", c.Name(), c.Short)
+			}
+		}
+	},
+}
+
+func init() {
+	clusterCmd.AddCommand(clusterCheckCmd)
+	clusterCmd.AddCommand(clusterCopiesCmd)
+	rootCmd.AddCommand(clusterCmd)
+
+	// Here you will define your flags and configuration settings.
+
+	// Cobra supports Persistent Flags which will work for this command
+	// and all subcommands, e.g.:
+	// nodeCmd.PersistentFlags().String("foo", "", "A help for foo")
+
+	// Cobra supports local flags which will only run when this command
+	// is called directly, e.g.:
+	// nodeCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle")
+}

sds-storage/cmd/cluster_check.go

@@ -0,0 +1,21 @@
+package cmd
+
+import (
+	"fmt"
+
+	"github.com/spf13/cobra"
+)
+
+var clusterCheckCmd = &cobra.Command{
+	Use:   "check",
+	Short: "A brief description of your command",
+	Long: `A longer description that spans multiple lines and likely contains examples
+and usage of using your command. For example:
+
+Cobra is a CLI library for Go that empowers applications.
+This application is a tool to generate the needed files
+to quickly create a Cobra application.`,
+	Run: func(cmd *cobra.Command, args []string) {
+		fmt.Println("cluster check called")
+	},
+}

sds-storage/cmd/cluster_copies.go

@@ -0,0 +1,21 @@
+package cmd
+
+import (
+	"fmt"
+
+	"github.com/spf13/cobra"
+)
+
+var clusterCopiesCmd = &cobra.Command{
+	Use:   "copies",
+	Short: "A brief description of your command",
+	Long: `A longer description that spans multiple lines and likely contains examples
+and usage of using your command. For example:
+
+Cobra is a CLI library for Go that empowers applications.
+This application is a tool to generate the needed files
+to quickly create a Cobra application.`,
+	Run: func(cmd *cobra.Command, args []string) {
+		fmt.Println("cluster copies called")
+	},
+}

sds-storage/cmd/dog.go

@@ -0,0 +1,36 @@
+package cmd
+
+import (
+	"fmt"
+
+	"github.com/spf13/cobra"
+)
+
+// dogCmd represents the dog command
+var dogCmd = &cobra.Command{
+	Use:   "dog",
+	Short: "Control sheepdog like",
+	Long: `A longer description that spans multiple lines and likely contains examples
+and usage of using your command. For example:
+
+Cobra is a CLI library for Go that empowers applications.
+This application is a tool to generate the needed files
+to quickly create a Cobra application.`,
+	Run: func(cmd *cobra.Command, args []string) {
+		fmt.Println("dog called")
+	},
+}
+
+func init() {
+	rootCmd.AddCommand(dogCmd)
+
+	// Here you will define your flags and configuration settings.
+
+	// Cobra supports Persistent Flags which will work for this command
+	// and all subcommands, e.g.:
+	// dogCmd.PersistentFlags().String("foo", "", "A help for foo")
+
+	// Cobra supports local flags which will only run when this command
+	// is called directly, e.g.:
+	// dogCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle")
+}

sds-storage/cmd/gateway.go

@@ -0,0 +1,36 @@
+package cmd
+
+import (
+	"fmt"
+
+	"github.com/spf13/cobra"
+)
+
+// gatewayCmd represents the gateway command
+var gatewayCmd = &cobra.Command{
+	Use:   "gateway",
+	Short: "Control gateway",
+	Long: `A longer description that spans multiple lines and likely contains examples
+and usage of using your command. For example:
+
+Cobra is a CLI library for Go that empowers applications.
+This application is a tool to generate the needed files
+to quickly create a Cobra application.`,
+	Run: func(cmd *cobra.Command, args []string) {
+		fmt.Println("gateway called")
+	},
+}
+
+func init() {
+	rootCmd.AddCommand(gatewayCmd)
+
+	// Here you will define your flags and configuration settings.
+
+	// Cobra supports Persistent Flags which will work for this command
+	// and all subcommands, e.g.:
+	// gatewayCmd.PersistentFlags().String("foo", "", "A help for foo")
+
+	// Cobra supports local flags which will only run when this command
+	// is called directly, e.g.:
+	// gatewayCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle")
+}

sds-storage/cmd/node.go

@@ -0,0 +1,50 @@
+package cmd
+
+import (
+	"fmt"
+
+	"github.com/spf13/cobra"
+)
+
+// nodeCmd represents the node command
+var nodeCmd = &cobra.Command{
+	Use:   "node",
+	Short: "Control node",
+	Long: `A longer description that spans multiple lines and likely contains examples
+and usage of using your command. For example:
+
+Cobra is a CLI library for Go that empowers applications.
+This application is a tool to generate the needed files
+to quickly create a Cobra application.`,
+	Run: func(cmd *cobra.Command, args []string) {
+		fmt.Println("node called")
+		fmt.Println("Available Commands:")
+		for _, c := range cmd.Commands() {
+			if c.IsAvailableCommand() {
+				fmt.Printf("  %s\t%s\n", c.Name(), c.Short)
+			}
+		}
+	},
+}
+
+func init() {
+	nodeCmd.AddCommand(nodeListCmd)
+	nodeCmd.AddCommand(nodeCheckCmd)
+	nodeCmd.AddCommand(nodePingCmd)
+	nodeCmd.AddCommand(nodeStoreCmd)
+	nodeCmd.AddCommand(nodeInfoCmd)
+	nodeCmd.AddCommand(nodeLogCmd)
+	nodeCmd.AddCommand(nodeKillCmd)
+	nodeCmd.AddCommand(nodeWeightCmd)
+	rootCmd.AddCommand(nodeCmd)
+
+	// Here you will define your flags and configuration settings.
+
+	// Cobra supports Persistent Flags which will work for this command
+	// and all subcommands, e.g.:
+	// nodeCmd.PersistentFlags().String("foo", "", "A help for foo")
+
+	// Cobra supports local flags which will only run when this command
+	// is called directly, e.g.:
+	// nodeCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle")
+}

sds-storage/cmd/node_check.go

@@ -0,0 +1,21 @@
+package cmd
+
+import (
+	"fmt"
+
+	"github.com/spf13/cobra"
+)
+
+var nodeCheckCmd = &cobra.Command{
+	Use:   "check",
+	Short: "A brief description of your command",
+	Long: `A longer description that spans multiple lines and likely contains examples
+and usage of using your command. For example:
+
+Cobra is a CLI library for Go that empowers applications.
+This application is a tool to generate the needed files
+to quickly create a Cobra application.`,
+	Run: func(cmd *cobra.Command, args []string) {
+		fmt.Println("node check called")
+	},
+}

sds-storage/cmd/node_info.go

@@ -0,0 +1,21 @@
+package cmd
+
+import (
+	"fmt"
+
+	"github.com/spf13/cobra"
+)
+
+var nodeInfoCmd = &cobra.Command{
+	Use:   "info",
+	Short: "A brief description of your command",
+	Long: `A longer description that spans multiple lines and likely contains examples
+and usage of using your command. For example:
+
+Cobra is a CLI library for Go that empowers applications.
+This application is a tool to generate the needed files
+to quickly create a Cobra application.`,
+	Run: func(cmd *cobra.Command, args []string) {
+		fmt.Println("node info called")
+	},
+}

sds-storage/cmd/node_kill.go

@@ -0,0 +1,21 @@
+package cmd
+
+import (
+	"fmt"
+
+	"github.com/spf13/cobra"
+)
+
+var nodeKillCmd = &cobra.Command{
+	Use:   "kill",
+	Short: "A brief description of your command",
+	Long: `A longer description that spans multiple lines and likely contains examples
+and usage of using your command. For example:
+
+Cobra is a CLI library for Go that empowers applications.
+This application is a tool to generate the needed files
+to quickly create a Cobra application.`,
+	Run: func(cmd *cobra.Command, args []string) {
+		fmt.Println("node kill called")
+	},
+}

sds-storage/cmd/node_list.go

@@ -0,0 +1,21 @@
+package cmd
+
+import (
+	"fmt"
+
+	"github.com/spf13/cobra"
+)
+
+var nodeListCmd = &cobra.Command{
+	Use:   "list",
+	Short: "A brief description of your command",
+	Long: `A longer description that spans multiple lines and likely contains examples
+and usage of using your command. For example:
+
+Cobra is a CLI library for Go that empowers applications.
+This application is a tool to generate the needed files
+to quickly create a Cobra application.`,
+	Run: func(cmd *cobra.Command, args []string) {
+		fmt.Println("node list called")
+	},
+}

sds-storage/cmd/node_log.go

@@ -0,0 +1,21 @@
+package cmd
+
+import (
+	"fmt"
+
+	"github.com/spf13/cobra"
+)
+
+var nodeLogCmd = &cobra.Command{
+	Use:   "log",
+	Short: "A brief description of your command",
+	Long: `A longer description that spans multiple lines and likely contains examples
+and usage of using your command. For example:
+
+Cobra is a CLI library for Go that empowers applications.
+This application is a tool to generate the needed files
+to quickly create a Cobra application.`,
+	Run: func(cmd *cobra.Command, args []string) {
+		fmt.Println("node log called")
+	},
+}

sds-storage/cmd/node_ping.go

@@ -0,0 +1,21 @@
+package cmd
+
+import (
+	"fmt"
+
+	"github.com/spf13/cobra"
+)
+
+var nodePingCmd = &cobra.Command{
+	Use:   "ping",
+	Short: "A brief description of your command",
+	Long: `A longer description that spans multiple lines and likely contains examples
+and usage of using your command. For example:
+
+Cobra is a CLI library for Go that empowers applications.
+This application is a tool to generate the needed files
+to quickly create a Cobra application.`,
+	Run: func(cmd *cobra.Command, args []string) {
+		fmt.Println("node ping called")
+	},
+}

sds-storage/cmd/node_store.go

@@ -0,0 +1,21 @@
+package cmd
+
+import (
+	"fmt"
+
+	"github.com/spf13/cobra"
+)
+
+var nodeStoreCmd = &cobra.Command{
+	Use:   "store",
+	Short: "A brief description of your command",
+	Long: `A longer description that spans multiple lines and likely contains examples
+and usage of using your command. For example:
+
+Cobra is a CLI library for Go that empowers applications.
+This application is a tool to generate the needed files
+to quickly create a Cobra application.`,
+	Run: func(cmd *cobra.Command, args []string) {
+		fmt.Println("node store called")
+	},
+}

sds-storage/cmd/node_weight.go

@@ -0,0 +1,21 @@
+package cmd
+
+import (
+	"fmt"
+
+	"github.com/spf13/cobra"
+)
+
+var nodeWeightCmd = &cobra.Command{
+	Use:   "weight",
+	Short: "A brief description of your command",
+	Long: `A longer description that spans multiple lines and likely contains examples
+and usage of using your command. For example:
+
+Cobra is a CLI library for Go that empowers applications.
+This application is a tool to generate the needed files
+to quickly create a Cobra application.`,
+	Run: func(cmd *cobra.Command, args []string) {
+		fmt.Println("node weight called")
+	},
+}

sds-storage/cmd/root.go

@@ -0,0 +1,79 @@
+package cmd
+
+import (
+	"fmt"
+	"os"
+
+	homedir "github.com/mitchellh/go-homedir"
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+)
+
+var cfgFile string
+
+// rootCmd represents the base command when called without any subcommands
+var rootCmd = &cobra.Command{
+	Use:   "storage",
+	Short: "A brief description of your application",
+	Long: `A longer description that spans multiple lines and likely contains
+examples and usage of using your application. For example:
+
+Cobra is a CLI library for Go that empowers applications.
+This application is a tool to generate the needed files
+to quickly create a Cobra application.`,
+	// Uncomment the following line if your bare application
+	// has an action associated with it:
+	//	Run: func(cmd *cobra.Command, args []string) { },
+}
+
+// Execute adds all child commands to the root command and sets flags appropriately.
+// This is called by main.main(). It only needs to happen once to the rootCmd.
+func Execute() {
+	if err := rootCmd.Execute(); err != nil {
+		fmt.Println(err)
+		os.Exit(1)
+	}
+}
+
+func init() {
+	cobra.OnInitialize(initConfig)
+
+	// Here you will define your flags and configuration settings.
+	// Cobra supports persistent flags, which, if defined here,
+	// will be global for your application.
+	rootCmd.PersistentFlags().StringVar(&cfgFile, "config", "", "config file (default is $HOME/.storage.yaml)")
+
+	// Cobra also supports local flags, which will only run
+	// when this action is called directly.
+	rootCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle")
+
+}
+
+// initConfig reads in config file and ENV variables if set.
+func initConfig() {
+	viper.SetConfigType("yaml")
+	if cfgFile != "" {
+		// Use config file from the flag.
+		viper.SetConfigFile(cfgFile)
+	} else {
+		// Find home directory.
+		home, err := homedir.Dir()
+		if err != nil {
+			fmt.Println(err)
+			os.Exit(1)
+		}
+
+		// Search config in home directory with name ".storage" (without extension).
+		viper.AddConfigPath("/etc/storage/")
+		viper.AddConfigPath(home)
+		viper.AddConfigPath(".")
+		viper.SetConfigName("storage")
+	}
+
+	viper.AutomaticEnv() // read in environment variables that match
+
+	// If a config file is found, read it in.
+	if err := viper.ReadInConfig(); err == nil {
+		fmt.Println("Using config file:", viper.ConfigFileUsed())
+	}
+}

sds-storage/cmd/server.go

@@ -0,0 +1,117 @@
+package cmd
+
+import (
+	"log"
+	"os"
+
+	"github.com/sdstack/storage/backend"
+	"github.com/sdstack/storage/cluster"
+	"github.com/sdstack/storage/kv"
+	"github.com/sdstack/storage/proxy"
+
+	// github.com/google/uuid
+
+	"net/http"
+	_ "net/http/pprof"
+
+	"github.com/spf13/cobra"
+	"github.com/spf13/viper"
+)
+
+// serverCmd represents the server command
+var serverCmd = &cobra.Command{
+	Use:   "server",
+	Short: "Control server",
+	Long: `A longer description that spans multiple lines and likely contains examples
+and usage of using your command. For example:
+
+Cobra is a CLI library for Go that empowers applications.
+This application is a tool to generate the needed files
+to quickly create a Cobra application.`,
+	Run: server,
+}
+
+func init() {
+	rootCmd.AddCommand(serverCmd)
+
+	// Here you will define your flags and configuration settings.
+
+	// Cobra supports Persistent Flags which will work for this command
+	// and all subcommands, e.g.:
+	// serverCmd.PersistentFlags().String("foo", "", "A help for foo")
+
+	// Cobra supports local flags which will only run when this command
+	// is called directly, e.g.:
+	// serverCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle")
+}
+
+func server(cmd *cobra.Command, args []string) {
+	if viper.GetBool("debug") {
+		go func() {
+			log.Println(http.ListenAndServe("localhost:6060", nil))
+		}()
+	}
+
+	var clusterEngine string
+	if viper.GetStringMap("cluster")["engine"] != nil {
+		clusterEngine = viper.GetStringMap("cluster")["engine"].(string)
+	}
+	if clusterEngine == "" {
+		clusterEngine = "none"
+	}
+	ce, err := cluster.New(clusterEngine, viper.GetStringMap("cluster")[clusterEngine])
+	if err != nil {
+		log.Printf("cluster start error %s", err)
+		os.Exit(1)
+	}
+
+	if err = ce.Start(); err != nil {
+		log.Printf("cluster start error %s", err)
+		os.Exit(1)
+	}
+	defer ce.Stop()
+
+	backendEngine := viper.GetStringMap("backend")["engine"].(string)
+	be, err := backend.New(backendEngine, viper.GetStringMap("backend")[backendEngine])
+	if err != nil {
+		log.Printf("store init error %s", err)
+		os.Exit(1)
+	}
+
+	engine, _ := kv.New(nil)
+	engine.SetBackend(be)
+	engine.SetCluster(ce)
+
+	for _, proxyEngine := range viper.GetStringMap("proxy")["engine"].([]interface{}) {
+		pe, err := proxy.New(proxyEngine.(string), viper.GetStringMap("proxy")[proxyEngine.(string)], engine)
+		if err != nil {
+			log.Printf("err: %s", err)
+			os.Exit(1)
+		}
+
+		if err = pe.Start(); err != nil {
+			log.Printf("err: %s", err)
+			os.Exit(1)
+		}
+		defer pe.Stop()
+		//conf.proxy = append(conf.proxy, proxy)
+	}
+
+	/*
+		for _, apiEngine := range viper.GetStringMap("api")["engine"].([]interface{}) {
+			ae, err := api.New(apiEngine.(string), viper.GetStringMap("api")[apiEngine.(string)], engine)
+			if err != nil {
+				log.Printf("err: %s", err)
+				os.Exit(1)
+			}
+
+			if err = ae.Start(); err != nil {
+				log.Printf("err: %s", err)
+				os.Exit(1)
+			}
+			defer ae.Stop()
+			//conf.proxy = append(conf.proxy, proxy)
+		}
+	*/
+	select {}
+}

sds-storage/cmd/volume.go

@@ -0,0 +1,58 @@
+package cmd
+
+import (
+	"fmt"
+
+	"github.com/spf13/cobra"
+)
+
+// volumeCmd represents the node command
+var volumeCmd = &cobra.Command{
+	Use:   "volume",
+	Short: "Control volumes",
+	Long: `A longer description that spans multiple lines and likely contains examples
+and usage of using your command. For example:
+
+Cobra is a CLI library for Go that empowers applications.
+This application is a tool to generate the needed files
+to quickly create a Cobra application.`,
+	Run: func(cmd *cobra.Command, args []string) {
+		fmt.Println("volume called")
+		fmt.Println("Available Commands:")
+		for _, c := range cmd.Commands() {
+			if c.IsAvailableCommand() {
+				fmt.Printf("  %s\t%s\n", c.Name(), c.Short)
+			}
+		}
+	},
+}
+
+func init() {
+	volumeCmd.AddCommand(volumeListCmd)
+	volumeCmd.AddCommand(volumeCheckCmd)
+	volumeCmd.AddCommand(volumeInfoCmd)
+	volumeCmd.AddCommand(volumeCreateCmd)
+	volumeCmd.AddCommand(volumeCloneCmd)
+	volumeCmd.AddCommand(volumeDeleteCmd)
+	volumeCmd.AddCommand(volumeRollbackCmd)
+	volumeCmd.AddCommand(volumeResizeCmd)
+	volumeCmd.AddCommand(volumeReadCmd)
+	volumeCmd.AddCommand(volumeWriteCmd)
+	volumeCmd.AddCommand(volumeExportCmd)
+	volumeCmd.AddCommand(volumeImportCmd)
+	volumeCmd.AddCommand(volumeLockCmd)
+	volumeCmd.AddCommand(volumeUnlockCmd)
+	volumeCmd.AddCommand(volumeCopiesCmd)
+	volumeCmd.AddCommand(volumeSnapshotCmd)
+	rootCmd.AddCommand(volumeCmd)
+
+	// Here you will define your flags and configuration settings.
+
+	// Cobra supports Persistent Flags which will work for this command
+	// and all subcommands, e.g.:
+	// nodeCmd.PersistentFlags().String("foo", "", "A help for foo")
+
+	// Cobra supports local flags which will only run when this command
+	// is called directly, e.g.:
+	// nodeCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle")
+}

sds-storage/cmd/volume_check.go

@@ -0,0 +1,21 @@
+package cmd
+
+import (
+	"fmt"
+
+	"github.com/spf13/cobra"
+)
+
+var volumeCheckCmd = &cobra.Command{
+	Use:   "list",
+	Short: "A brief description of your command",
+	Long: `A longer description that spans multiple lines and likely contains examples
+and usage of using your command. For example:
+
+Cobra is a CLI library for Go that empowers applications.
+This application is a tool to generate the needed files
+to quickly create a Cobra application.`,
+	Run: func(cmd *cobra.Command, args []string) {
+		fmt.Println("volume check called")
+	},
+}

sds-storage/cmd/volume_clone.go

@@ -0,0 +1,21 @@
+package cmd
+
+import (
+	"fmt"
+
+	"github.com/spf13/cobra"
+)
+
+var volumeCloneCmd = &cobra.Command{
+	Use:   "clone",
+	Short: "A brief description of your command",
+	Long: `A longer description that spans multiple lines and likely contains examples
+and usage of using your command. For example:
+
+Cobra is a CLI library for Go that empowers applications.
+This application is a tool to generate the needed files
+to quickly create a Cobra application.`,
+	Run: func(cmd *cobra.Command, args []string) {
+		fmt.Println("volume clone called")
+	},
+}

sds-storage/cmd/volume_copies.go

@@ -0,0 +1,21 @@
+package cmd
+
+import (
+	"fmt"
+
+	"github.com/spf13/cobra"
+)
+
+var volumeCopiesCmd = &cobra.Command{
+	Use:   "copies",
+	Short: "A brief description of your command",
+	Long: `A longer description that spans multiple lines and likely contains examples
+and usage of using your command. For example:
+
+Cobra is a CLI library for Go that empowers applications.
+This application is a tool to generate the needed files
+to quickly create a Cobra application.`,
+	Run: func(cmd *cobra.Command, args []string) {
+		fmt.Println("volume copies called")
+	},
+}

sds-storage/cmd/volume_create.go

@@ -0,0 +1,21 @@
+package cmd
+
+import (
+	"fmt"
+
+	"github.com/spf13/cobra"
+)
+
+var volumeCreateCmd = &cobra.Command{
+	Use:   "create",
+	Short: "A brief description of your command",
+	Long: `A longer description that spans multiple lines and likely contains examples
+and usage of using your command. For example:
+
+Cobra is a CLI library for Go that empowers applications.
+This application is a tool to generate the needed files
+to quickly create a Cobra application.`,
+	Run: func(cmd *cobra.Command, args []string) {
+		fmt.Println("volume create called")
+	},
+}

sds-storage/cmd/volume_delete.go

@@ -0,0 +1,21 @@
+package cmd
+
+import (
+	"fmt"
+
+	"github.com/spf13/cobra"
+)
+
+var volumeDeleteCmd = &cobra.Command{
+	Use:   "delete",
+	Short: "A brief description of your command",
+	Long: `A longer description that spans multiple lines and likely contains examples
+and usage of using your command. For example:
+
+Cobra is a CLI library for Go that empowers applications.
+This application is a tool to generate the needed files
+to quickly create a Cobra application.`,
+	Run: func(cmd *cobra.Command, args []string) {
+		fmt.Println("volume delete called")
+	},
+}

sds-storage/cmd/volume_export.go

@@ -0,0 +1,21 @@
+package cmd
+
+import (
+	"fmt"
+
+	"github.com/spf13/cobra"
+)
+
+var volumeExportCmd = &cobra.Command{
+	Use:   "export",
+	Short: "A brief description of your command",
+	Long: `A longer description that spans multiple lines and likely contains examples
+and usage of using your command. For example:
+
+Cobra is a CLI library for Go that empowers applications.
+This application is a tool to generate the needed files
+to quickly create a Cobra application.`,
+	Run: func(cmd *cobra.Command, args []string) {
+		fmt.Println("volume export called")
+	},
+}

sds-storage/cmd/volume_import.go

@@ -0,0 +1,21 @@
+package cmd
+
+import (
+	"fmt"
+
+	"github.com/spf13/cobra"
+)
+
+var volumeImportCmd = &cobra.Command{
+	Use:   "import",
+	Short: "A brief description of your command",
+	Long: `A longer description that spans multiple lines and likely contains examples
+and usage of using your command. For example:
+
+Cobra is a CLI library for Go that empowers applications.
+This application is a tool to generate the needed files
+to quickly create a Cobra application.`,
+	Run: func(cmd *cobra.Command, args []string) {
+		fmt.Println("volume import called")
+	},
+}

sds-storage/cmd/volume_info.go

@@ -0,0 +1,21 @@
+package cmd
+
+import (
+	"fmt"
+
+	"github.com/spf13/cobra"
+)
+
+var volumeInfoCmd = &cobra.Command{
+	Use:   "info",
+	Short: "A brief description of your command",
+	Long: `A longer description that spans multiple lines and likely contains examples
+and usage of using your command. For example:
+
+Cobra is a CLI library for Go that empowers applications.
+This application is a tool to generate the needed files
+to quickly create a Cobra application.`,
+	Run: func(cmd *cobra.Command, args []string) {
+		fmt.Println("volume info called")
+	},
+}

sds-storage/cmd/volume_list.go

@@ -0,0 +1,21 @@
+package cmd
+
+import (
+	"fmt"
+
+	"github.com/spf13/cobra"
+)
+
+var volumeListCmd = &cobra.Command{
+	Use:   "list",
+	Short: "A brief description of your command",
+	Long: `A longer description that spans multiple lines and likely contains examples
+and usage of using your command. For example:
+
+Cobra is a CLI library for Go that empowers applications.
+This application is a tool to generate the needed files
+to quickly create a Cobra application.`,
+	Run: func(cmd *cobra.Command, args []string) {
+		fmt.Println("volume list called")
+	},
+}

sds-storage/cmd/volume_lock.go

@@ -0,0 +1,21 @@
+package cmd
+
+import (
+	"fmt"
+
+	"github.com/spf13/cobra"
+)
+
+var volumeLockCmd = &cobra.Command{
+	Use:   "lock",
+	Short: "A brief description of your command",
+	Long: `A longer description that spans multiple lines and likely contains examples
+and usage of using your command. For example:
+
+Cobra is a CLI library for Go that empowers applications.
+This application is a tool to generate the needed files
+to quickly create a Cobra application.`,
+	Run: func(cmd *cobra.Command, args []string) {
+		fmt.Println("volume lock called")
+	},
+}

sds-storage/cmd/volume_read.go

@@ -0,0 +1,21 @@
+package cmd
+
+import (
+	"fmt"
+
+	"github.com/spf13/cobra"
+)
+
+var volumeReadCmd = &cobra.Command{
+	Use:   "read",
+	Short: "A brief description of your command",
+	Long: `A longer description that spans multiple lines and likely contains examples
+and usage of using your command. For example:
+
+Cobra is a CLI library for Go that empowers applications.
+This application is a tool to generate the needed files
+to quickly create a Cobra application.`,
+	Run: func(cmd *cobra.Command, args []string) {
+		fmt.Println("volume read called")
+	},
+}

sds-storage/cmd/volume_resize.go

@@ -0,0 +1,21 @@
+package cmd
+
+import (
+	"fmt"
+
+	"github.com/spf13/cobra"
+)
+
+var volumeResizeCmd = &cobra.Command{
+	Use:   "resize",
+	Short: "A brief description of your command",
+	Long: `A longer description that spans multiple lines and likely contains examples
+and usage of using your command. For example:
+
+Cobra is a CLI library for Go that empowers applications.
+This application is a tool to generate the needed files
+to quickly create a Cobra application.`,
+	Run: func(cmd *cobra.Command, args []string) {
+		fmt.Println("volume resize called")
+	},
+}

sds-storage/cmd/volume_rollback.go

@@ -0,0 +1,21 @@
+package cmd
+
+import (
+	"fmt"
+
+	"github.com/spf13/cobra"
+)
+
+var volumeRollbackCmd = &cobra.Command{
+	Use:   "rollback",
+	Short: "A brief description of your command",
+	Long: `A longer description that spans multiple lines and likely contains examples
+and usage of using your command. For example:
+
+Cobra is a CLI library for Go that empowers applications.
+This application is a tool to generate the needed files
+to quickly create a Cobra application.`,
+	Run: func(cmd *cobra.Command, args []string) {
+		fmt.Println("volume rollback called")
+	},
+}

sds-storage/cmd/volume_snapshot.go

@@ -0,0 +1,21 @@
+package cmd
+
+import (
+	"fmt"
+
+	"github.com/spf13/cobra"
+)
+
+var volumeSnapshotCmd = &cobra.Command{
+	Use:   "snapshot",
+	Short: "A brief description of your command",
+	Long: `A longer description that spans multiple lines and likely contains examples
+and usage of using your command. For example:
+
+Cobra is a CLI library for Go that empowers applications.
+This application is a tool to generate the needed files
+to quickly create a Cobra application.`,
+	Run: func(cmd *cobra.Command, args []string) {
+		fmt.Println("volume snapshot called")
+	},
+}

sds-storage/cmd/volume_unlock.go

@@ -0,0 +1,21 @@
+package cmd
+
+import (
+	"fmt"
+
+	"github.com/spf13/cobra"
+)
+
+var volumeUnlockCmd = &cobra.Command{
+	Use:   "unlock",
+	Short: "A brief description of your command",
+	Long: `A longer description that spans multiple lines and likely contains examples
+and usage of using your command. For example:
+
+Cobra is a CLI library for Go that empowers applications.
+This application is a tool to generate the needed files
+to quickly create a Cobra application.`,
+	Run: func(cmd *cobra.Command, args []string) {
+		fmt.Println("volume unlock called")
+	},
+}

sds-storage/cmd/volume_write.go

@@ -0,0 +1,21 @@
+package cmd
+
+import (
+	"fmt"
+
+	"github.com/spf13/cobra"
+)
+
+var volumeWriteCmd = &cobra.Command{
+	Use:   "write",
+	Short: "A brief description of your command",
+	Long: `A longer description that spans multiple lines and likely contains examples
+and usage of using your command. For example:
+
+Cobra is a CLI library for Go that empowers applications.
+This application is a tool to generate the needed files
+to quickly create a Cobra application.`,
+	Run: func(cmd *cobra.Command, args []string) {
+		fmt.Println("volume write called")
+	},
+}

sds-storage/hash_xxhash.go

@@ -0,0 +1,7 @@
+// +build hash_xxhash
+
+package main
+
+import (
+	_ "github.com/sdstack/storage/hash/xxhash"
+)

sds-storage/main.go

@@ -0,0 +1,7 @@
+package main // import "github.com/sdstack/storage"
+
+import "github.com/sdstack/storage/sds-storage/cmd"
+
+func main() {
+	cmd.Execute()
+}

sds-storage/proxy_sheepdog.go

@@ -0,0 +1,7 @@
+// +build proxy_sheepdog
+
+package main
+
+import (
+	_ "github.com/sdstack/storage/proxy/sheepdog"
+)

sds-storage/storage.yml

@@ -0,0 +1,44 @@
+debug: true
+
+discovery:
+  engine: mdns
+  mdns:
+    zone: local
+    interface: ib0
+
+cluster:
+  engine: none
+  etcdint:
+    debug: true
+    server_addr: 172.16.1.254:2380
+    client_addr: 172.16.1.254:2379
+    wal_size: 18874368
+    store: data/cluster/etcdint
+
+proxy:
+  engine: [ sheepdog ]
+  sheepdog:
+    debug: true
+    maxconn: 10240
+    listen:
+      - tcp://172.16.1.254:7000
+      - unix://var/run/sheepdog.sock
+
+api:
+  engine: [ json, msgpack ]
+  json:
+    listen:
+      - unix://var/run/storage.sock
+
+backend:
+  engine: filesystem
+  filesystem:
+    debug: true
+    options:
+      sync: true
+
+node:
+  name: cc.z1.sdstack.com
+  uuid: 1234567-89-00-99-99
+  zone: zoneuuid
+  rack: rackuuid

tests/fio.txt

@@ -0,0 +1,15 @@
+[global]
+filename=/path/to/device
+runtime=120
+ioengine=libaio
+direct=1
+ramp_time=10    
+group_reporting
+
+[randrw]
+readwrite=randrw
+rwmixread=70
+rwmixwrite=30
+iodepth=4
+blocksize=4k
+

transport/transport.go

@@ -0,0 +1,27 @@
+package transport
+
+import (
+	"errors"
+	"net"
+	"time"
+)
+
+func DialTimeout(network string, address string, timeout time.Duration) (net.Conn, error) {
+	switch network {
+	//	case "utp":
+	//	return utp.DialTimeout(address, timeout)
+	case "tcp":
+		return net.DialTimeout("tcp", address, timeout)
+	}
+	return nil, errors.New("unsupported network")
+}
+
+func Listen(network string, laddr string) (net.Listener, error) {
+	switch network {
+	//	case "utp":
+	//	return utp.NewSocket("udp", laddr)
+	case "tcp":
+		return net.Listen(network, laddr)
+	}
+	return nil, errors.New("unsupported network")
+}