micro/store/cloudflare/cloudflare.go

409 lines
9.6 KiB
Go
Raw Normal View History

// Package cloudflare is a store implementation backed by cloudflare workers kv
// Note that the cloudflare workers KV API is eventually consistent.
package cloudflare
import (
"bytes"
"context"
"encoding/json"
"fmt"
"io"
"io/ioutil"
"log"
"math"
"net/http"
"net/url"
2019-10-24 01:10:44 +03:00
"os"
"strconv"
2020-03-24 17:51:43 +03:00
"strings"
"time"
"github.com/micro/go-micro/v2/store"
"github.com/pkg/errors"
"github.com/patrickmn/go-cache"
)
2019-10-24 00:54:55 +03:00
const (
apiBaseURL = "https://api.cloudflare.com/client/v4/"
)
type workersKV struct {
2019-12-16 17:38:51 +03:00
options store.Options
2019-10-24 00:51:08 +03:00
// cf account id
account string
// cf api token
token string
// cf kv namespace
namespace string
// http client to use
httpClient *http.Client
// cache
cache *cache.Cache
}
2019-10-24 00:57:11 +03:00
// apiResponse is a cloudflare v4 api response
type apiResponse struct {
Result []struct {
ID string `json:"id"`
Type string `json:"type"`
Name string `json:"name"`
Expiration int64 `json:"expiration"`
2019-10-24 00:57:11 +03:00
Content string `json:"content"`
Proxiable bool `json:"proxiable"`
Proxied bool `json:"proxied"`
TTL int64 `json:"ttl"`
Priority int64 `json:"priority"`
2019-10-24 00:57:11 +03:00
Locked bool `json:"locked"`
ZoneID string `json:"zone_id"`
ZoneName string `json:"zone_name"`
ModifiedOn time.Time `json:"modified_on"`
CreatedOn time.Time `json:"created_on"`
} `json:"result"`
Success bool `json:"success"`
Errors []apiMessage `json:"errors"`
// not sure Messages is ever populated?
Messages []apiMessage `json:"messages"`
ResultInfo struct {
Page int `json:"page"`
PerPage int `json:"per_page"`
Count int `json:"count"`
TotalCount int `json:"total_count"`
} `json:"result_info"`
}
// apiMessage is a Cloudflare v4 API Error
type apiMessage struct {
Code int `json:"code"`
Message string `json:"message"`
}
2019-10-24 01:10:44 +03:00
// getOptions returns account id, token and namespace
func getOptions() (string, string, string) {
2020-03-24 17:51:43 +03:00
accountID := strings.TrimSpace(os.Getenv("CF_ACCOUNT_ID"))
apiToken := strings.TrimSpace(os.Getenv("CF_API_TOKEN"))
namespace := strings.TrimSpace(os.Getenv("KV_NAMESPACE_ID"))
2019-10-24 01:10:44 +03:00
return accountID, apiToken, namespace
}
func validateOptions(account, token, namespace string) {
if len(account) == 0 {
log.Fatal("Store: CF_ACCOUNT_ID is blank")
}
if len(token) == 0 {
log.Fatal("Store: CF_API_TOKEN is blank")
}
if len(namespace) == 0 {
log.Fatal("Store: KV_NAMESPACE_ID is blank")
}
}
func (w *workersKV) Close() error {
return nil
}
func (w *workersKV) Init(opts ...store.Option) error {
for _, o := range opts {
o(&w.options)
}
if len(w.options.Database) > 0 {
w.namespace = w.options.Database
}
ttl := w.options.Context.Value("STORE_CACHE_TTL")
if ttl != nil {
2020-03-03 16:15:26 +03:00
ttlduration, ok := ttl.(time.Duration)
if !ok {
log.Fatal("STORE_CACHE_TTL from context must be type int64")
}
w.cache = cache.New(ttlduration, 3*ttlduration)
}
return nil
}
func (w *workersKV) list(prefix string) ([]string, error) {
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
2019-10-24 00:51:08 +03:00
path := fmt.Sprintf("accounts/%s/storage/kv/namespaces/%s/keys", w.account, w.namespace)
body := make(map[string]string)
if len(prefix) > 0 {
body["prefix"] = prefix
}
response, _, _, err := w.request(ctx, http.MethodGet, path, body, make(http.Header))
if err != nil {
return nil, err
}
2019-10-24 00:51:08 +03:00
a := &apiResponse{}
if err := json.Unmarshal(response, a); err != nil {
return nil, err
}
2019-10-24 00:51:08 +03:00
if !a.Success {
messages := ""
for _, m := range a.Errors {
messages += strconv.Itoa(m.Code) + " " + m.Message + "\n"
}
return nil, errors.New(messages)
}
keys := make([]string, 0, len(a.Result))
2019-10-24 00:51:08 +03:00
for _, r := range a.Result {
keys = append(keys, r.Name)
}
2019-10-24 00:51:08 +03:00
return keys, nil
}
// In the cloudflare workers KV implemention, List() doesn't guarantee
// anything as the workers API is eventually consistent.
func (w *workersKV) List(opts ...store.ListOption) ([]string, error) {
keys, err := w.list("")
if err != nil {
return nil, err
}
return keys, nil
}
func (w *workersKV) Read(key string, opts ...store.ReadOption) ([]*store.Record, error) {
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
var options store.ReadOptions
for _, o := range opts {
o(&options)
}
keys := []string{key}
if options.Prefix {
k, err := w.list(key)
if err != nil {
return nil, err
}
keys = k
}
//nolint:prealloc
var records []*store.Record
2019-10-24 00:51:08 +03:00
for _, k := range keys {
if w.cache != nil {
if resp, hit := w.cache.Get(k); hit {
if record, ok := resp.(*store.Record); ok {
records = append(records, record)
continue
}
}
}
2019-10-24 00:51:08 +03:00
path := fmt.Sprintf("accounts/%s/storage/kv/namespaces/%s/values/%s", w.account, w.namespace, url.PathEscape(k))
response, headers, status, err := w.request(ctx, http.MethodGet, path, nil, make(http.Header))
if err != nil {
return records, err
}
if status < 200 || status >= 300 {
2020-03-12 02:09:42 +03:00
if status == 404 {
return nil, store.ErrNotFound
}
return records, errors.New("Received unexpected Status " + strconv.Itoa(status) + string(response))
}
record := &store.Record{
Key: k,
Value: response,
}
if expiry := headers.Get("Expiration"); len(expiry) != 0 {
expiryUnix, err := strconv.ParseInt(expiry, 10, 64)
if err != nil {
return records, err
}
record.Expiry = time.Until(time.Unix(expiryUnix, 0))
}
2020-03-03 16:54:56 +03:00
if w.cache != nil {
w.cache.Set(record.Key, record, cache.DefaultExpiration)
}
records = append(records, record)
}
2019-10-24 00:51:08 +03:00
return records, nil
}
func (w *workersKV) Write(r *store.Record, opts ...store.WriteOption) error {
// Set it in local cache, with the global TTL from options
if w.cache != nil {
w.cache.Set(r.Key, r, cache.DefaultExpiration)
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
path := fmt.Sprintf("accounts/%s/storage/kv/namespaces/%s/values/%s", w.account, w.namespace, url.PathEscape(r.Key))
if r.Expiry != 0 {
// Minimum cloudflare TTL is 60 Seconds
exp := int(math.Max(60, math.Round(r.Expiry.Seconds())))
path = path + "?expiration_ttl=" + strconv.Itoa(exp)
}
2019-10-24 00:51:08 +03:00
headers := make(http.Header)
2019-10-24 00:51:08 +03:00
resp, _, _, err := w.request(ctx, http.MethodPut, path, r.Value, headers)
if err != nil {
return err
}
2019-10-24 00:51:08 +03:00
a := &apiResponse{}
if err := json.Unmarshal(resp, a); err != nil {
return err
}
2019-10-24 00:51:08 +03:00
if !a.Success {
messages := ""
for _, m := range a.Errors {
messages += strconv.Itoa(m.Code) + " " + m.Message + "\n"
}
return errors.New(messages)
}
2019-10-24 00:51:08 +03:00
return nil
}
func (w *workersKV) Delete(key string, opts ...store.DeleteOption) error {
2020-03-03 16:15:26 +03:00
if w.cache != nil {
w.cache.Delete(key)
2020-03-03 16:15:26 +03:00
}
ctx, cancel := context.WithTimeout(context.Background(), 30*time.Second)
defer cancel()
path := fmt.Sprintf("accounts/%s/storage/kv/namespaces/%s/values/%s", w.account, w.namespace, url.PathEscape(key))
resp, _, _, err := w.request(ctx, http.MethodDelete, path, nil, make(http.Header))
if err != nil {
return err
}
a := &apiResponse{}
if err := json.Unmarshal(resp, a); err != nil {
return err
}
2019-10-24 00:51:08 +03:00
if !a.Success {
messages := ""
for _, m := range a.Errors {
messages += strconv.Itoa(m.Code) + " " + m.Message + "\n"
}
return errors.New(messages)
}
2019-10-24 00:51:08 +03:00
return nil
}
func (w *workersKV) request(ctx context.Context, method, path string, body interface{}, headers http.Header) ([]byte, http.Header, int, error) {
var jsonBody []byte
var err error
if body != nil {
if paramBytes, ok := body.([]byte); ok {
jsonBody = paramBytes
} else {
jsonBody, err = json.Marshal(body)
if err != nil {
return nil, nil, 0, errors.Wrap(err, "error marshalling params to JSON")
}
}
} else {
jsonBody = nil
}
2019-10-24 00:51:08 +03:00
var reqBody io.Reader
2019-10-24 00:51:08 +03:00
if jsonBody != nil {
reqBody = bytes.NewReader(jsonBody)
}
2019-10-24 00:51:08 +03:00
req, err := http.NewRequestWithContext(ctx, method, apiBaseURL+path, reqBody)
if err != nil {
return nil, nil, 0, errors.Wrap(err, "error creating new request")
}
for key, value := range headers {
req.Header[key] = value
}
2019-10-24 00:51:08 +03:00
// set token if it exists
if len(w.token) > 0 {
req.Header.Set("Authorization", "Bearer "+w.token)
}
2019-10-24 00:51:08 +03:00
// set the user agent to micro
req.Header.Set("User-Agent", "micro/1.0 (https://micro.mu)")
// Official cloudflare client does exponential backoff here
2019-10-24 00:51:08 +03:00
// TODO: retry and use util/backoff
resp, err := w.httpClient.Do(req)
if err != nil {
return nil, nil, 0, err
}
defer resp.Body.Close()
2019-10-24 00:51:08 +03:00
respBody, err := ioutil.ReadAll(resp.Body)
if err != nil {
return respBody, resp.Header, resp.StatusCode, err
}
2019-10-24 00:51:08 +03:00
return respBody, resp.Header, resp.StatusCode, nil
}
func (w *workersKV) String() string {
return "cloudflare"
}
func (w *workersKV) Options() store.Options {
return w.options
}
// NewStore returns a cloudflare Store implementation.
2019-10-24 01:10:44 +03:00
// Account ID, Token and Namespace must either be passed as options or
// environment variables. If set as env vars we expect the following;
// CF_API_TOKEN to a cloudflare API token scoped to Workers KV.
// CF_ACCOUNT_ID to contain a string with your cloudflare account ID.
2019-10-24 00:54:55 +03:00
// KV_NAMESPACE_ID to contain the namespace UUID for your KV storage.
2019-12-16 17:38:51 +03:00
func NewStore(opts ...store.Option) store.Store {
var options store.Options
for _, o := range opts {
o(&options)
}
2019-10-24 00:54:55 +03:00
2019-12-16 17:38:51 +03:00
// get options from environment
2019-10-24 01:10:44 +03:00
account, token, namespace := getOptions()
2019-10-24 00:54:55 +03:00
2019-12-16 17:38:51 +03:00
if len(account) == 0 {
account = getAccount(options.Context)
2019-10-24 00:54:55 +03:00
}
2019-12-16 17:38:51 +03:00
if len(token) == 0 {
token = getToken(options.Context)
2019-10-24 00:54:55 +03:00
}
2019-12-16 17:38:51 +03:00
if len(namespace) == 0 {
namespace = options.Database
2019-10-24 00:54:55 +03:00
}
2019-10-24 01:10:44 +03:00
// validate options are not blank or log.Fatal
validateOptions(account, token, namespace)
2019-10-24 00:54:55 +03:00
return &workersKV{
account: account,
namespace: namespace,
token: token,
2019-12-16 17:38:51 +03:00
options: options,
2019-10-24 00:54:55 +03:00
httpClient: &http.Client{},
}
}