Merge pull request #861 from micro/certmagicstorage

Distributed storage for certmagic

.travis.yml

@@ -2,7 +2,7 @@ language: go
 go:
 - 1.13.x
 env:
-  - GO111MODULE=on
+  - GO111MODULE=on IN_TRAVIS_CI=yes
 notifications:
   slack:
     secure: aEvhLbhujaGaKSrOokiG3//PaVHTIrc3fBpoRbCRqfZpyq6WREoapJJhF+tIpWWOwaC9GmChbD6aHo/jMUgwKXVyPSaNjiEL87YzUUpL8B2zslNp1rgfTg/LrzthOx3Q1TYwpaAl3to0fuHUVFX4yMeC2vuThq7WSXgMMxFCtbc=

api/server/acme/acme.go

@@ -4,8 +4,6 @@ package acme
 import (
 	"errors"
 	"net"
-
-	"github.com/go-acme/lego/v3/challenge"
 )
 
 var (
@@ -24,62 +22,3 @@ const (
 	LetsEncryptStagingCA    = "https://acme-staging-v02.api.letsencrypt.org/directory"
 	LetsEncryptProductionCA = "https://acme-v02.api.letsencrypt.org/directory"
 )
-
-// Option (or Options) are passed to New() to configure providers
-type Option func(o *Options)
-
-// Options represents various options you can present to ACME providers
-type Options struct {
-	// AcceptTLS must be set to true to indicate that you have read your
-	// provider's terms of service.
-	AcceptToS bool
-	// CA is the CA to use
-	CA string
-	// ChallengeProvider is a go-acme/lego challenge provider. Set this if you
-	// want to use DNS Challenges. Otherwise, tls-alpn-01 will be used
-	ChallengeProvider challenge.Provider
-	// Issue certificates for domains on demand. Otherwise, certs will be
-	// retrieved / issued on start-up.
-	OnDemand bool
-	// TODO
-	Cache interface{}
-}
-
-// AcceptToS indicates whether you accept your CA's terms of service
-func AcceptToS(b bool) Option {
-	return func(o *Options) {
-		o.AcceptToS = b
-	}
-}
-
-// CA sets the CA of an acme.Options
-func CA(CA string) Option {
-	return func(o *Options) {
-		o.CA = CA
-	}
-}
-
-// ChallengeProvider sets the Challenge provider of an acme.Options
-// if set, it enables the DNS challenge, otherwise tls-alpn-01 will be used.
-func ChallengeProvider(p challenge.Provider) Option {
-	return func(o *Options) {
-		o.ChallengeProvider = p
-	}
-}
-
-// OnDemand enables on-demand certificate issuance. Not recommended for use
-// with the DNS challenge, as the first connection may be very slow.
-func OnDemand(b bool) Option {
-	return func(o *Options) {
-		o.OnDemand = b
-	}
-}
-
-// Default uses the Let's Encrypt Production CA, with DNS Challenge disabled.
-func Default() []Option {
-	return []Option{
-		AcceptToS(true),
-		CA(LetsEncryptProductionCA),
-		OnDemand(true),
-	}
-}

api/server/acme/certmagic/certmagic.go

@@ -2,6 +2,7 @@
 package certmagic
 
 import (
+	"log"
 	"net"
 
 	"github.com/mholt/certmagic"
@@ -36,6 +37,12 @@ func New(options ...acme.Option) acme.Provider {
 			op(o)
 		}
 	}
+	if o.Cache != nil {
+		if _, ok := o.Cache.(certmagic.Storage); !ok {
+			log.Fatal("ACME: cache provided doesn't implement certmagic's Storage interface")
+		}
+	}
+
 	return &certmagicProvider{
 		opts: o,
 	}

api/server/acme/certmagic/certmagic_test.go

@@ -1,19 +1,29 @@
 package certmagic
 
 import (
+	"os"
+	"reflect"
+	"sort"
 	"testing"
+	"time"
 
 	"github.com/go-acme/lego/v3/providers/dns/cloudflare"
-	// "github.com/micro/go-micro/api/server/acme"
+	"github.com/mholt/certmagic"
+	"github.com/micro/go-micro/api/server/acme"
+	"github.com/micro/go-micro/config/options"
+	cloudflarestorage "github.com/micro/go-micro/store/cloudflare"
+	"github.com/micro/go-micro/sync/lock/memory"
 )
 
 func TestCertMagic(t *testing.T) {
-	// TODO: Travis doesn't let us bind :443
+	if len(os.Getenv("IN_TRAVIS_CI")) != 0 {
-	// l, err := New().NewListener()
+		t.Skip("Travis doesn't let us bind :443")
-	// if err != nil {
+	}
-	// 	t.Error(err.Error())
+	l, err := New().NewListener()
-	// }
+	if err != nil {
-	// l.Close()
+		t.Error(err.Error())
+	}
+	l.Close()
 
 	c := cloudflare.NewDefaultConfig()
 	c.AuthEmail = ""
@@ -21,19 +31,157 @@ func TestCertMagic(t *testing.T) {
 	c.AuthToken = "test"
 	c.ZoneToken = "test"
 
-	_, err := cloudflare.NewDNSProviderConfig(c)
+	p, err := cloudflare.NewDNSProviderConfig(c)
 	if err != nil {
 		t.Error(err.Error())
 	}
 
-	// TODO: Travis doesn't let us bind :443
+	l, err = New(acme.AcceptToS(true),
-	// l, err = New(acme.AcceptTLS(true),
+		acme.CA(acme.LetsEncryptStagingCA),
-	// 	acme.CA(acme.LetsEncryptStagingCA),
+		acme.ChallengeProvider(p),
-	// 	acme.ChallengeProvider(p),
+	).NewListener()
-	// ).NewListener()
 
-	// if err != nil {
+	if err != nil {
-	// 	t.Error(err.Error())
+		t.Error(err.Error())
-	// }
+	}
-	// l.Close()
+	l.Close()
+}
+
+func TestStorageImplementation(t *testing.T) {
+	apiToken, accountID := os.Getenv("CF_API_TOKEN"), os.Getenv("CF_ACCOUNT_ID")
+	kvID := os.Getenv("KV_NAMESPACE_ID")
+	if len(apiToken) == 0 || len(accountID) == 0 || len(kvID) == 0 {
+		t.Skip("No Cloudflare API keys available, skipping test")
+	}
+
+	var s certmagic.Storage
+	st, err := cloudflarestorage.New(
+		options.WithValue("CF_API_TOKEN", apiToken),
+		options.WithValue("CF_ACCOUNT_ID", accountID),
+		options.WithValue("KV_NAMESPACE_ID", kvID),
+	)
+	if err != nil {
+		t.Fatalf("Couldn't initialise cloudflare storage: %s\n", err.Error())
+	}
+	s = &storage{
+		lock:  memory.NewLock(),
+		store: st,
+	}
+
+	// Test Lock
+	if err := s.Lock("test"); err != nil {
+		t.Error(err)
+	}
+
+	// Test Unlock
+	if err := s.Unlock("test"); err != nil {
+		t.Error(err)
+	}
+
+	// Test data
+	testdata := []struct {
+		key   string
+		value []byte
+	}{
+		{key: "/foo/a", value: []byte("lorem")},
+		{key: "/foo/b", value: []byte("ipsum")},
+		{key: "/foo/c", value: []byte("dolor")},
+		{key: "/foo/d", value: []byte("sit")},
+		{key: "/bar/a", value: []byte("amet")},
+		{key: "/bar/b", value: []byte("consectetur")},
+		{key: "/bar/c", value: []byte("adipiscing")},
+		{key: "/bar/d", value: []byte("elit")},
+		{key: "/foo/bar/a", value: []byte("sed")},
+		{key: "/foo/bar/b", value: []byte("do")},
+		{key: "/foo/bar/c", value: []byte("eiusmod")},
+		{key: "/foo/bar/d", value: []byte("tempor")},
+		{key: "/foo/bar/baz/a", value: []byte("incididunt")},
+		{key: "/foo/bar/baz/b", value: []byte("ut")},
+		{key: "/foo/bar/baz/c", value: []byte("labore")},
+		{key: "/foo/bar/baz/d", value: []byte("et")},
+		// a duplicate just in case there's any edge cases
+		{key: "/foo/a", value: []byte("lorem")},
+	}
+
+	// Test Store
+	for _, d := range testdata {
+		if err := s.Store(d.key, d.value); err != nil {
+			t.Error(err.Error())
+		}
+	}
+
+	// Test Load
+	for _, d := range testdata {
+		if value, err := s.Load(d.key); err != nil {
+			t.Error(err.Error())
+		} else {
+			if !reflect.DeepEqual(value, d.value) {
+				t.Errorf("Load %s: expected %v, got %v", d.key, d.value, value)
+			}
+		}
+	}
+
+	// Test Exists
+	for _, d := range testdata {
+		if !s.Exists(d.key) {
+			t.Errorf("%s should exist, but doesn't\n", d.key)
+		}
+	}
+
+	// Test List
+	if list, err := s.List("/", true); err != nil {
+		t.Error(err.Error())
+	} else {
+		var expected []string
+		for i, d := range testdata {
+			if i != len(testdata)-1 {
+				// Don't store the intentionally duplicated key
+				expected = append(expected, d.key)
+			}
+		}
+		sort.Strings(expected)
+		sort.Strings(list)
+		if !reflect.DeepEqual(expected, list) {
+			t.Errorf("List: Expected %v, got %v\n", expected, list)
+		}
+	}
+	if list, err := s.List("/foo", false); err != nil {
+		t.Error(err.Error())
+	} else {
+		sort.Strings(list)
+		expected := []string{"/foo/a", "/foo/b", "/foo/bar", "/foo/c", "/foo/d"}
+		if !reflect.DeepEqual(expected, list) {
+			t.Errorf("List: expected %s, got %s\n", expected, list)
+		}
+	}
+
+	// Test Stat
+	for _, d := range testdata {
+		info, err := s.Stat(d.key)
+		if err != nil {
+			t.Error(err.Error())
+		} else {
+			if info.Key != d.key {
+				t.Errorf("Stat().Key: expected %s, got %s\n", d.key, info.Key)
+			}
+			if info.Size != int64(len(d.value)) {
+				t.Errorf("Stat().Size: expected %d, got %d\n", len(d.value), info.Size)
+			}
+			if time.Since(info.Modified) > time.Minute {
+				t.Errorf("Stat().Modified: expected time since last modified to be < 1 minute, got %v\n", time.Since(info.Modified))
+			}
+		}
+
+	}
+
+	// Test Delete
+	for _, d := range testdata {
+		if err := s.Delete(d.key); err != nil {
+			t.Error(err.Error())
+		}
+	}
+
+	// New interface doesn't return an error, so call it in case any log.Fatal
+	// happens
+	New(acme.Cache(s))
 }

api/server/acme/certmagic/storage.go

@@ -0,0 +1,134 @@
+package certmagic
+
+import (
+	"bytes"
+	"encoding/gob"
+	"fmt"
+	"path"
+	"strings"
+	"time"
+
+	"github.com/mholt/certmagic"
+	"github.com/micro/go-micro/store"
+	"github.com/micro/go-micro/sync/lock"
+)
+
+// File represents a "File" that will be stored in store.Store - the contents and last modified time
+type File struct {
+	// last modified time
+	LastModified time.Time
+	// Contents
+	Contents []byte
+}
+
+// storage is an implementation of certmagic.Storage using micro's sync.Map and store.Store interfaces.
+// As certmagic storage expects a filesystem (with stat() abilities) we have to implement
+// the bare minimum of metadata.
+type storage struct {
+	lock  lock.Lock
+	store store.Store
+}
+
+func (s *storage) Lock(key string) error {
+	return s.lock.Acquire(key, lock.TTL(10*time.Minute))
+}
+
+func (s *storage) Unlock(key string) error {
+	return s.lock.Release(key)
+}
+
+func (s *storage) Store(key string, value []byte) error {
+	f := File{
+		LastModified: time.Now(),
+		Contents:     value,
+	}
+	buf := &bytes.Buffer{}
+	e := gob.NewEncoder(buf)
+	if err := e.Encode(f); err != nil {
+		return err
+	}
+	r := &store.Record{
+		Key:   key,
+		Value: buf.Bytes(),
+	}
+	return s.store.Write(r)
+}
+
+func (s *storage) Load(key string) ([]byte, error) {
+	records, err := s.store.Read(key)
+	if err != nil {
+		return nil, err
+	}
+	if len(records) != 1 {
+		return nil, fmt.Errorf("ACME Storage: multiple records matched key %s", key)
+	}
+	b := bytes.NewBuffer(records[0].Value)
+	d := gob.NewDecoder(b)
+	var f File
+	err = d.Decode(&f)
+	if err != nil {
+		return nil, err
+	}
+	return f.Contents, nil
+}
+
+func (s *storage) Delete(key string) error {
+	return s.store.Delete(key)
+}
+
+func (s *storage) Exists(key string) bool {
+	_, err := s.store.Read()
+	if err != nil {
+		return false
+	}
+	return true
+}
+
+func (s *storage) List(prefix string, recursive bool) ([]string, error) {
+	records, err := s.store.Sync()
+	if err != nil {
+		return nil, err
+	}
+	var results []string
+	for _, r := range records {
+		if strings.HasPrefix(r.Key, prefix) {
+			results = append(results, r.Key)
+		}
+	}
+	if recursive {
+		return results, nil
+	}
+	keysMap := make(map[string]bool)
+	for _, key := range results {
+		dir := strings.Split(strings.TrimPrefix(key, prefix+"/"), "/")
+		keysMap[dir[0]] = true
+	}
+	results = make([]string, 0)
+	for k := range keysMap {
+		results = append(results, path.Join(prefix, k))
+	}
+	return results, nil
+}
+
+func (s *storage) Stat(key string) (certmagic.KeyInfo, error) {
+	records, err := s.store.Read(key)
+	if err != nil {
+		return certmagic.KeyInfo{}, err
+	}
+	if len(records) != 1 {
+		return certmagic.KeyInfo{}, fmt.Errorf("ACME Storage: multiple records matched key %s", key)
+	}
+	b := bytes.NewBuffer(records[0].Value)
+	d := gob.NewDecoder(b)
+	var f File
+	err = d.Decode(&f)
+	if err != nil {
+		return certmagic.KeyInfo{}, err
+	}
+	return certmagic.KeyInfo{
+		Key:        key,
+		Modified:   f.LastModified,
+		Size:       int64(len(f.Contents)),
+		IsTerminal: false,
+	}, nil
+}

api/server/acme/options.go

@@ -0,0 +1,73 @@
+package acme
+
+import "github.com/go-acme/lego/v3/challenge"
+
+// Option (or Options) are passed to New() to configure providers
+type Option func(o *Options)
+
+// Options represents various options you can present to ACME providers
+type Options struct {
+	// AcceptTLS must be set to true to indicate that you have read your
+	// provider's terms of service.
+	AcceptToS bool
+	// CA is the CA to use
+	CA string
+	// ChallengeProvider is a go-acme/lego challenge provider. Set this if you
+	// want to use DNS Challenges. Otherwise, tls-alpn-01 will be used
+	ChallengeProvider challenge.Provider
+	// Issue certificates for domains on demand. Otherwise, certs will be
+	// retrieved / issued on start-up.
+	OnDemand bool
+	// Cache is a storage interface. Most ACME libraries have an cache, but
+	// there's no defined interface, so if you consume this option
+	// sanity check it before using.
+	Cache interface{}
+}
+
+// AcceptToS indicates whether you accept your CA's terms of service
+func AcceptToS(b bool) Option {
+	return func(o *Options) {
+		o.AcceptToS = b
+	}
+}
+
+// CA sets the CA of an acme.Options
+func CA(CA string) Option {
+	return func(o *Options) {
+		o.CA = CA
+	}
+}
+
+// ChallengeProvider sets the Challenge provider of an acme.Options
+// if set, it enables the DNS challenge, otherwise tls-alpn-01 will be used.
+func ChallengeProvider(p challenge.Provider) Option {
+	return func(o *Options) {
+		o.ChallengeProvider = p
+	}
+}
+
+// OnDemand enables on-demand certificate issuance. Not recommended for use
+// with the DNS challenge, as the first connection may be very slow.
+func OnDemand(b bool) Option {
+	return func(o *Options) {
+		o.OnDemand = b
+	}
+}
+
+// Cache provides a cache / storage interface to the underlying ACME library
+// as there is no standard, this needs to be validated by the underlying
+// implentation.
+func Cache(c interface{}) Option {
+	return func(o *Options) {
+		o.Cache = c
+	}
+}
+
+// Default uses the Let's Encrypt Production CA, with DNS Challenge disabled.
+func Default() []Option {
+	return []Option{
+		AcceptToS(true),
+		CA(LetsEncryptProductionCA),
+		OnDemand(true),
+	}
+}