micro/store/cockroach/cockroach.go

386 lines
8.7 KiB
Go
Raw Normal View History

2019-12-16 15:09:59 +00:00
// Package cockroach implements the cockroach store
package cockroach
2019-11-01 14:13:21 +00:00
import (
"database/sql"
"fmt"
"net/url"
"regexp"
2019-12-16 15:09:59 +00:00
"strings"
2019-11-01 14:13:21 +00:00
"time"
"github.com/lib/pq"
"github.com/micro/go-micro/v2/store"
2019-12-16 14:38:51 +00:00
"github.com/pkg/errors"
2019-11-01 14:13:21 +00:00
)
// DefaultDatabase is the namespace that the sql store
2019-11-01 14:13:21 +00:00
// will use if no namespace is provided.
var (
DefaultDatabase = "micro"
DefaultTable = "micro"
)
2019-11-01 14:13:21 +00:00
type sqlStore struct {
db *sql.DB
database string
table string
2019-12-16 14:38:51 +00:00
list *sql.Stmt
readOne *sql.Stmt
readMany *sql.Stmt
readOffset *sql.Stmt
write *sql.Stmt
delete *sql.Stmt
2019-12-16 14:38:51 +00:00
options store.Options
2019-11-01 14:13:21 +00:00
}
func (s *sqlStore) Close() error {
closeStmt(s.delete)
closeStmt(s.list)
closeStmt(s.readMany)
closeStmt(s.readOffset)
closeStmt(s.readOne)
closeStmt(s.write)
if s.db != nil {
return s.db.Close()
}
return nil
}
func (s *sqlStore) Init(opts ...store.Option) error {
for _, o := range opts {
o(&s.options)
}
// reconfigure
return s.configure()
}
2019-11-01 14:13:21 +00:00
// List all the known records
func (s *sqlStore) List(opts ...store.ListOption) ([]string, error) {
rows, err := s.list.Query()
var keys []string
2019-11-01 14:13:21 +00:00
var timehelper pq.NullTime
if err != nil {
if err == sql.ErrNoRows {
return keys, nil
2019-11-01 14:13:21 +00:00
}
return nil, err
}
defer rows.Close()
for rows.Next() {
record := &store.Record{}
if err := rows.Scan(&record.Key, &record.Value, &timehelper); err != nil {
return keys, err
2019-11-01 14:13:21 +00:00
}
if timehelper.Valid {
if timehelper.Time.Before(time.Now()) {
// record has expired
go s.Delete(record.Key)
} else {
record.Expiry = time.Until(timehelper.Time)
keys = append(keys, record.Key)
2019-11-01 14:13:21 +00:00
}
} else {
keys = append(keys, record.Key)
2019-11-01 14:13:21 +00:00
}
}
rowErr := rows.Close()
if rowErr != nil {
// transaction rollback or something
return keys, rowErr
2019-11-01 14:13:21 +00:00
}
if err := rows.Err(); err != nil {
return keys, err
2019-11-01 14:13:21 +00:00
}
return keys, nil
2019-11-01 14:13:21 +00:00
}
// Read a single key
func (s *sqlStore) Read(key string, opts ...store.ReadOption) ([]*store.Record, error) {
var options store.ReadOptions
for _, o := range opts {
o(&options)
}
if options.Prefix || options.Suffix {
return s.read(key, options)
}
2019-11-01 14:13:21 +00:00
var records []*store.Record
var timehelper pq.NullTime
row := s.readOne.QueryRow(key)
record := &store.Record{}
if err := row.Scan(&record.Key, &record.Value, &timehelper); err != nil {
if err == sql.ErrNoRows {
return records, store.ErrNotFound
2019-11-01 14:13:21 +00:00
}
return records, err
}
if timehelper.Valid {
if timehelper.Time.Before(time.Now()) {
// record has expired
go s.Delete(key)
return records, store.ErrNotFound
2019-11-01 14:13:21 +00:00
}
record.Expiry = time.Until(timehelper.Time)
records = append(records, record)
} else {
records = append(records, record)
2019-11-01 14:13:21 +00:00
}
2019-11-01 14:13:21 +00:00
return records, nil
}
// Read Many records
func (s *sqlStore) read(key string, options store.ReadOptions) ([]*store.Record, error) {
pattern := "%"
if options.Prefix {
pattern = key + pattern
}
if options.Suffix {
pattern = pattern + key
}
var rows *sql.Rows
var err error
if options.Limit != 0 {
rows, err = s.readOffset.Query(pattern, options.Limit, options.Offset)
} else {
rows, err = s.readMany.Query(pattern)
}
if err != nil {
if err == sql.ErrNoRows {
return []*store.Record{}, nil
}
return []*store.Record{}, errors.Wrap(err, "sqlStore.read failed")
}
defer rows.Close()
var records []*store.Record
var timehelper pq.NullTime
for rows.Next() {
record := &store.Record{}
if err := rows.Scan(&record.Key, &record.Value, &timehelper); err != nil {
return records, err
}
if timehelper.Valid {
if timehelper.Time.Before(time.Now()) {
// record has expired
go s.Delete(record.Key)
} else {
record.Expiry = time.Until(timehelper.Time)
records = append(records, record)
}
} else {
records = append(records, record)
}
}
rowErr := rows.Close()
if rowErr != nil {
// transaction rollback or something
return records, rowErr
}
if err := rows.Err(); err != nil {
return records, err
}
return records, nil
}
2019-11-01 14:13:21 +00:00
// Write records
func (s *sqlStore) Write(r *store.Record, opts ...store.WriteOption) error {
var err error
if r.Expiry != 0 {
_, err = s.write.Exec(r.Key, r.Value, time.Now().Add(r.Expiry))
} else {
_, err = s.write.Exec(r.Key, r.Value, nil)
}
if err != nil {
return errors.Wrap(err, "Couldn't insert record "+r.Key)
2019-11-01 14:13:21 +00:00
}
return nil
}
// Delete records with keys
func (s *sqlStore) Delete(key string, opts ...store.DeleteOption) error {
result, err := s.delete.Exec(key)
if err != nil {
return err
2019-11-01 14:13:21 +00:00
}
_, err = result.RowsAffected()
if err != nil {
return err
}
2019-11-01 14:13:21 +00:00
return nil
}
func (s *sqlStore) initDB() error {
2019-12-16 17:11:13 +00:00
// Create the namespace's database
_, err := s.db.Exec(fmt.Sprintf("CREATE DATABASE IF NOT EXISTS %s ;", s.database))
2019-11-01 14:13:21 +00:00
if err != nil {
return err
2019-11-01 14:13:21 +00:00
}
2019-12-16 14:38:51 +00:00
2019-12-16 17:11:13 +00:00
_, err = s.db.Exec(fmt.Sprintf("SET DATABASE = %s ;", s.database))
2019-11-01 14:13:21 +00:00
if err != nil {
return errors.Wrap(err, "Couldn't set database")
2019-11-01 14:13:21 +00:00
}
2019-12-16 17:11:13 +00:00
// Create a table for the namespace's prefix
_, err = s.db.Exec(fmt.Sprintf(`CREATE TABLE IF NOT EXISTS %s
2019-11-01 14:13:21 +00:00
(
2019-12-16 17:11:13 +00:00
key text NOT NULL,
2019-11-01 14:13:21 +00:00
value bytea,
expiry timestamp with time zone,
CONSTRAINT %s_pkey PRIMARY KEY (key)
2019-12-16 17:11:13 +00:00
);`, s.table, s.table))
2019-11-01 14:13:21 +00:00
if err != nil {
return errors.Wrap(err, "Couldn't create table")
2019-11-01 14:13:21 +00:00
}
// Create Index
2020-03-24 23:49:09 +00:00
_, err = s.db.Exec(fmt.Sprintf(`CREATE INDEX IF NOT EXISTS "%s" ON %s.%s USING btree ("key");`, "key_index_"+s.table, s.database, s.table))
if err != nil {
return err
}
list, err := s.db.Prepare(fmt.Sprintf("SELECT key, value, expiry FROM %s.%s;", s.database, s.table))
if err != nil {
return errors.Wrap(err, "List statement couldn't be prepared")
}
closeStmt(s.list)
s.list = list
readOne, err := s.db.Prepare(fmt.Sprintf("SELECT key, value, expiry FROM %s.%s WHERE key = $1;", s.database, s.table))
if err != nil {
return errors.Wrap(err, "ReadOne statement couldn't be prepared")
}
closeStmt(s.readOne)
s.readOne = readOne
readMany, err := s.db.Prepare(fmt.Sprintf("SELECT key, value, expiry FROM %s.%s WHERE key LIKE $1;", s.database, s.table))
if err != nil {
return errors.Wrap(err, "ReadMany statement couldn't be prepared")
}
closeStmt(s.readMany)
s.readMany = readMany
readOffset, err := s.db.Prepare(fmt.Sprintf("SELECT key, value, expiry FROM %s.%s WHERE key LIKE $1 ORDER BY key DESC LIMIT $2 OFFSET $3;", s.database, s.table))
if err != nil {
return errors.Wrap(err, "ReadOffset statement couldn't be prepared")
}
closeStmt(s.readOffset)
s.readOffset = readOffset
write, err := s.db.Prepare(fmt.Sprintf(`INSERT INTO %s.%s(key, value, expiry)
VALUES ($1, $2::bytea, $3)
ON CONFLICT (key)
DO UPDATE
SET value = EXCLUDED.value, expiry = EXCLUDED.expiry;`, s.database, s.table))
if err != nil {
return errors.Wrap(err, "Write statement couldn't be prepared")
}
closeStmt(s.write)
s.write = write
delete, err := s.db.Prepare(fmt.Sprintf("DELETE FROM %s.%s WHERE key = $1;", s.database, s.table))
if err != nil {
return errors.Wrap(err, "Delete statement couldn't be prepared")
}
closeStmt(s.delete)
s.delete = delete
return nil
}
2019-11-01 14:13:21 +00:00
func (s *sqlStore) configure() error {
if len(s.options.Nodes) == 0 {
s.options.Nodes = []string{"postgresql://root@localhost:26257?sslmode=disable"}
2019-12-16 14:38:51 +00:00
}
2019-11-01 14:13:21 +00:00
database := s.options.Database
if len(database) == 0 {
database = DefaultDatabase
2019-11-01 14:13:21 +00:00
}
2019-12-16 14:38:51 +00:00
table := s.options.Table
if len(table) == 0 {
table = DefaultTable
}
2019-12-16 14:38:51 +00:00
// store.namespace must only contain letters, numbers and underscores
reg, err := regexp.Compile("[^a-zA-Z0-9]+")
if err != nil {
return errors.New("error compiling regex for namespace")
2019-11-01 14:13:21 +00:00
}
database = reg.ReplaceAllString(database, "_")
table = reg.ReplaceAllString(table, "_")
2019-12-16 14:38:51 +00:00
source := s.options.Nodes[0]
// check if it is a standard connection string eg: host=%s port=%d user=%s password=%s dbname=%s sslmode=disable
// if err is nil which means it would be a URL like postgre://xxxx?yy=zz
_, err = url.Parse(source)
if err != nil {
if !strings.Contains(source, " ") {
source = fmt.Sprintf("host=%s", source)
}
2019-12-16 15:09:59 +00:00
}
2019-12-16 14:38:51 +00:00
// create source from first node
2019-12-16 15:09:59 +00:00
db, err := sql.Open("postgres", source)
2019-12-16 14:38:51 +00:00
if err != nil {
return err
2019-11-01 14:13:21 +00:00
}
2019-12-16 14:38:51 +00:00
if err := db.Ping(); err != nil {
return err
2019-11-01 14:13:21 +00:00
}
2019-12-16 14:38:51 +00:00
if s.db != nil {
s.db.Close()
2019-11-01 14:13:21 +00:00
}
// save the values
s.db = db
s.database = database
s.table = table
// initialise the database
return s.initDB()
}
func (s *sqlStore) String() string {
return "cockroach"
}
func (s *sqlStore) Options() store.Options {
return s.options
}
// NewStore returns a new micro Store backed by sql
func NewStore(opts ...store.Option) store.Store {
var options store.Options
for _, o := range opts {
o(&options)
}
// new store
s := new(sqlStore)
// set the options
s.options = options
2020-03-24 17:16:38 +00:00
// best-effort configure the store
s.configure()
// return store
2019-12-16 14:38:51 +00:00
return s
2019-11-01 14:13:21 +00:00
}
func closeStmt(s *sql.Stmt) {
if s != nil {
s.Close()
}
}