WIP: Add metadata to store record (#1604)

* Add metadata to store record

* Add metadata to cockroach store

* add metadata to store service implementation

* fix breaking cache test

* Test/fix cockroach metadata usage

* fix store memory metadata bug

store/cockroach/cockroach.go

@@ -27,11 +27,11 @@ var (
 	re = regexp.MustCompile("[^a-zA-Z0-9]+")
 
 	statements = map[string]string{
-		"list":       "SELECT key, value, expiry FROM %s.%s;",
-		"read":       "SELECT key, value, expiry FROM %s.%s WHERE key = $1;",
-		"readMany":   "SELECT key, value, expiry FROM %s.%s WHERE key LIKE $1;",
-		"readOffset": "SELECT key, value, expiry FROM %s.%s WHERE key LIKE $1 ORDER BY key DESC LIMIT $2 OFFSET $3;",
-		"write":      "INSERT INTO %s.%s(key, value, expiry) VALUES ($1, $2::bytea, $3) ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value, expiry = EXCLUDED.expiry;",
+		"list":       "SELECT key, value, metadata, expiry FROM %s.%s;",
+		"read":       "SELECT key, value, metadata, expiry FROM %s.%s WHERE key = $1;",
+		"readMany":   "SELECT key, value, metadata, expiry FROM %s.%s WHERE key LIKE $1;",
+		"readOffset": "SELECT key, value, metadata, expiry FROM %s.%s WHERE key LIKE $1 ORDER BY key DESC LIMIT $2 OFFSET $3;",
+		"write":      "INSERT INTO %s.%s(key, value, metadata, expiry) VALUES ($1, $2::bytea, $3, $4) ON CONFLICT (key) DO UPDATE SET value = EXCLUDED.value, metadata = EXCLUDED.metadata, expiry = EXCLUDED.expiry;",
 		"delete":     "DELETE FROM %s.%s WHERE key = $1;",
 	}
 )
@@ -108,6 +108,7 @@ func (s *sqlStore) initDB(database, table string) error {
 	(
 		key text NOT NULL,
 		value bytea,
+		metadata JSONB,
 		expiry timestamp with time zone,
 		CONSTRAINT %s_pkey PRIMARY KEY (key)
 	);`, table, table))
@@ -121,6 +122,12 @@ func (s *sqlStore) initDB(database, table string) error {
 		return err
 	}
 
+	// Create Metadata Index
+	_, err = s.db.Exec(fmt.Sprintf(`CREATE INDEX IF NOT EXISTS "%s" ON %s.%s USING GIN ("metadata");`, "metadata_index_"+table, database, table))
+	if err != nil {
+		return err
+	}
+
 	return nil
 }
 
@@ -227,9 +234,15 @@ func (s *sqlStore) List(opts ...store.ListOption) ([]string, error) {
 
 	for rows.Next() {
 		record := &store.Record{}
-		if err := rows.Scan(&record.Key, &record.Value, &timehelper); err != nil {
+		metadata := make(Metadata)
+
+		if err := rows.Scan(&record.Key, &record.Value, &metadata, &timehelper); err != nil {
 			return keys, err
 		}
+
+		// set the metadata
+		record.Metadata = toMetadata(&metadata)
+
 		if timehelper.Valid {
 			if timehelper.Time.Before(time.Now()) {
 				// record has expired
@@ -281,12 +294,18 @@ func (s *sqlStore) Read(key string, opts ...store.ReadOption) ([]*store.Record,
 
 	row := st.QueryRow(key)
 	record := &store.Record{}
-	if err := row.Scan(&record.Key, &record.Value, &timehelper); err != nil {
+	metadata := make(Metadata)
+
+	if err := row.Scan(&record.Key, &record.Value, &metadata, &timehelper); err != nil {
 		if err == sql.ErrNoRows {
 			return records, store.ErrNotFound
 		}
 		return records, err
 	}
+
+	// set the metadata
+	record.Metadata = toMetadata(&metadata)
+
 	if timehelper.Valid {
 		if timehelper.Time.Before(time.Now()) {
 			// record has expired
@@ -346,9 +365,15 @@ func (s *sqlStore) read(key string, options store.ReadOptions) ([]*store.Record,
 
 	for rows.Next() {
 		record := &store.Record{}
-		if err := rows.Scan(&record.Key, &record.Value, &timehelper); err != nil {
+		metadata := make(Metadata)
+
+		if err := rows.Scan(&record.Key, &record.Value, &metadata, &timehelper); err != nil {
 			return records, err
 		}
+
+		// set the metadata
+		record.Metadata = toMetadata(&metadata)
+
 		if timehelper.Valid {
 			if timehelper.Time.Before(time.Now()) {
 				// record has expired
@@ -391,10 +416,15 @@ func (s *sqlStore) Write(r *store.Record, opts ...store.WriteOption) error {
 	}
 	defer st.Close()
 
+	metadata := make(Metadata)
+	for k, v := range r.Metadata {
+		metadata[k] = v
+	}
+
 	if r.Expiry != 0 {
-		_, err = st.Exec(r.Key, r.Value, time.Now().Add(r.Expiry))
+		_, err = st.Exec(r.Key, r.Value, metadata, time.Now().Add(r.Expiry))
 	} else {
-		_, err = st.Exec(r.Key, r.Value, nil)
+		_, err = st.Exec(r.Key, r.Value, metadata, nil)
 	}
 
 	if err != nil {

store/cockroach/metadata.go

@@ -0,0 +1,45 @@
+package cockroach
+
+import (
+	"database/sql/driver"
+	"encoding/json"
+	"errors"
+)
+
+// https://github.com/upper/db/blob/master/postgresql/custom_types.go#L43
+type Metadata map[string]interface{}
+
+// Scan satisfies the sql.Scanner interface.
+func (m *Metadata) Scan(src interface{}) error {
+	source, ok := src.([]byte)
+	if !ok {
+		return errors.New("Type assertion .([]byte) failed.")
+	}
+
+	var i interface{}
+	err := json.Unmarshal(source, &i)
+	if err != nil {
+		return err
+	}
+
+	*m, ok = i.(map[string]interface{})
+	if !ok {
+		return errors.New("Type assertion .(map[string]interface{}) failed.")
+	}
+
+	return nil
+}
+
+// Value satisfies the driver.Valuer interface.
+func (m Metadata) Value() (driver.Value, error) {
+	j, err := json.Marshal(m)
+	return j, err
+}
+
+func toMetadata(m *Metadata) map[string]interface{} {
+	md := make(map[string]interface{})
+	for k, v := range *m {
+		md[k] = v
+	}
+	return md
+}