From a6e95d389fc2b5971e88e09b8426b5dbc7bc8e50 Mon Sep 17 00:00:00 2001 From: Jake Sanders Date: Tue, 15 Oct 2019 19:32:20 +0100 Subject: [PATCH 1/2] Implementation of certmagic storage using micro's store and sync packages --- .travis.yml | 2 +- api/server/acme/acme.go | 61 --------- api/server/acme/certmagic/certmagic.go | 7 + api/server/acme/certmagic/certmagic_test.go | 50 +++++--- api/server/acme/certmagic/storage.go | 134 ++++++++++++++++++++ api/server/acme/options.go | 73 +++++++++++ 6 files changed, 248 insertions(+), 79 deletions(-) create mode 100644 api/server/acme/certmagic/storage.go create mode 100644 api/server/acme/options.go diff --git a/.travis.yml b/.travis.yml index cb760ae9..e2a018e2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -2,7 +2,7 @@ language: go go: - 1.13.x env: - - GO111MODULE=on + - GO111MODULE=on IN_TRAVIS_CI=yes notifications: slack: secure: aEvhLbhujaGaKSrOokiG3//PaVHTIrc3fBpoRbCRqfZpyq6WREoapJJhF+tIpWWOwaC9GmChbD6aHo/jMUgwKXVyPSaNjiEL87YzUUpL8B2zslNp1rgfTg/LrzthOx3Q1TYwpaAl3to0fuHUVFX4yMeC2vuThq7WSXgMMxFCtbc= diff --git a/api/server/acme/acme.go b/api/server/acme/acme.go index cecf49e0..f9df8962 100644 --- a/api/server/acme/acme.go +++ b/api/server/acme/acme.go @@ -4,8 +4,6 @@ package acme import ( "errors" "net" - - "github.com/go-acme/lego/v3/challenge" ) var ( @@ -24,62 +22,3 @@ const ( LetsEncryptStagingCA = "https://acme-staging-v02.api.letsencrypt.org/directory" LetsEncryptProductionCA = "https://acme-v02.api.letsencrypt.org/directory" ) - -// Option (or Options) are passed to New() to configure providers -type Option func(o *Options) - -// Options represents various options you can present to ACME providers -type Options struct { - // AcceptTLS must be set to true to indicate that you have read your - // provider's terms of service. - AcceptToS bool - // CA is the CA to use - CA string - // ChallengeProvider is a go-acme/lego challenge provider. Set this if you - // want to use DNS Challenges. Otherwise, tls-alpn-01 will be used - ChallengeProvider challenge.Provider - // Issue certificates for domains on demand. Otherwise, certs will be - // retrieved / issued on start-up. - OnDemand bool - // TODO - Cache interface{} -} - -// AcceptToS indicates whether you accept your CA's terms of service -func AcceptToS(b bool) Option { - return func(o *Options) { - o.AcceptToS = b - } -} - -// CA sets the CA of an acme.Options -func CA(CA string) Option { - return func(o *Options) { - o.CA = CA - } -} - -// ChallengeProvider sets the Challenge provider of an acme.Options -// if set, it enables the DNS challenge, otherwise tls-alpn-01 will be used. -func ChallengeProvider(p challenge.Provider) Option { - return func(o *Options) { - o.ChallengeProvider = p - } -} - -// OnDemand enables on-demand certificate issuance. Not recommended for use -// with the DNS challenge, as the first connection may be very slow. -func OnDemand(b bool) Option { - return func(o *Options) { - o.OnDemand = b - } -} - -// Default uses the Let's Encrypt Production CA, with DNS Challenge disabled. -func Default() []Option { - return []Option{ - AcceptToS(true), - CA(LetsEncryptProductionCA), - OnDemand(true), - } -} diff --git a/api/server/acme/certmagic/certmagic.go b/api/server/acme/certmagic/certmagic.go index 68e36f64..6d12e9d6 100644 --- a/api/server/acme/certmagic/certmagic.go +++ b/api/server/acme/certmagic/certmagic.go @@ -2,6 +2,7 @@ package certmagic import ( + "log" "net" "github.com/mholt/certmagic" @@ -36,6 +37,12 @@ func New(options ...acme.Option) acme.Provider { op(o) } } + if o.Cache != nil { + if _, ok := o.Cache.(certmagic.Storage); !ok { + log.Fatal("ACME: cache provided doesn't implement certmagic's Storage interface") + } + } + return &certmagicProvider{ opts: o, } diff --git a/api/server/acme/certmagic/certmagic_test.go b/api/server/acme/certmagic/certmagic_test.go index dc727101..52ede19c 100644 --- a/api/server/acme/certmagic/certmagic_test.go +++ b/api/server/acme/certmagic/certmagic_test.go @@ -1,19 +1,24 @@ package certmagic import ( + "os" "testing" "github.com/go-acme/lego/v3/providers/dns/cloudflare" - // "github.com/micro/go-micro/api/server/acme" + "github.com/mholt/certmagic" + "github.com/micro/go-micro/api/server/acme" + "github.com/micro/go-micro/sync/lock/memory" ) func TestCertMagic(t *testing.T) { - // TODO: Travis doesn't let us bind :443 - // l, err := New().NewListener() - // if err != nil { - // t.Error(err.Error()) - // } - // l.Close() + if len(os.Getenv("IN_TRAVIS_CI")) != 0 { + t.Skip("Travis doesn't let us bind :443") + } + l, err := New().NewListener() + if err != nil { + t.Error(err.Error()) + } + l.Close() c := cloudflare.NewDefaultConfig() c.AuthEmail = "" @@ -21,19 +26,30 @@ func TestCertMagic(t *testing.T) { c.AuthToken = "test" c.ZoneToken = "test" - _, err := cloudflare.NewDNSProviderConfig(c) + p, err := cloudflare.NewDNSProviderConfig(c) if err != nil { t.Error(err.Error()) } - // TODO: Travis doesn't let us bind :443 - // l, err = New(acme.AcceptTLS(true), - // acme.CA(acme.LetsEncryptStagingCA), - // acme.ChallengeProvider(p), - // ).NewListener() + l, err = New(acme.AcceptToS(true), + acme.CA(acme.LetsEncryptStagingCA), + acme.ChallengeProvider(p), + ).NewListener() - // if err != nil { - // t.Error(err.Error()) - // } - // l.Close() + if err != nil { + t.Error(err.Error()) + } + l.Close() +} + +func TestStorageImplementation(t *testing.T) { + var s certmagic.Storage + s = &storage{ + lock: memory.NewLock(), + } + if err := s.Lock("test"); err != nil { + t.Error(err) + } + s.Unlock("test") + New(acme.Cache(s)) } diff --git a/api/server/acme/certmagic/storage.go b/api/server/acme/certmagic/storage.go new file mode 100644 index 00000000..b1bb92b4 --- /dev/null +++ b/api/server/acme/certmagic/storage.go @@ -0,0 +1,134 @@ +package certmagic + +import ( + "bytes" + "encoding/gob" + "fmt" + "path" + "strings" + "time" + + "github.com/mholt/certmagic" + "github.com/micro/go-micro/store" + "github.com/micro/go-micro/sync/lock" +) + +// file represents a "file" that will be stored in store.Store - the contents and last modified time +type file struct { + // last modified time + lastModified time.Time + // contents + contents []byte +} + +// storage is an implementation of certmagic.Storage using micro's sync.Map and store.Store interfaces. +// As certmagic storage expects a filesystem (with stat() abilities) we have to implement +// the bare minimum of metadata. +type storage struct { + lock lock.Lock + store store.Store +} + +func (s *storage) Lock(key string) error { + return s.lock.Acquire(key, lock.TTL(10*time.Minute)) +} + +func (s *storage) Unlock(key string) error { + return s.lock.Release(key) +} + +func (s *storage) Store(key string, value []byte) error { + f := file{ + lastModified: time.Now(), + contents: value, + } + buf := &bytes.Buffer{} + e := gob.NewEncoder(buf) + if err := e.Encode(f); err != nil { + return err + } + r := &store.Record{ + Key: key, + Value: buf.Bytes(), + } + return s.store.Write(r) +} + +func (s *storage) Load(key string) ([]byte, error) { + records, err := s.store.Read(key) + if err != nil { + return nil, err + } + if len(records) != 1 { + return nil, fmt.Errorf("ACME Storage: multiple records matched key %s", key) + } + b := bytes.NewBuffer(records[0].Value) + d := gob.NewDecoder(b) + var f file + err = d.Decode(&f) + if err != nil { + return nil, err + } + return f.contents, nil +} + +func (s *storage) Delete(key string) error { + return s.store.Delete(key) +} + +func (s *storage) Exists(key string) bool { + _, err := s.store.Read() + if err != nil { + return false + } + return true +} + +func (s *storage) List(prefix string, recursive bool) ([]string, error) { + records, err := s.store.Sync() + if err != nil { + return nil, err + } + var results []string + for _, r := range records { + if strings.HasPrefix(r.Key, prefix) { + results = append(results, r.Key) + } + } + if recursive { + return results, nil + } + keysMap := make(map[string]bool) + for _, key := range results { + dir := strings.Split(strings.TrimPrefix(key, prefix+"/"), "/") + keysMap[dir[0]] = true + } + results = make([]string, 0) + for k := range keysMap { + results = append(results, path.Join(prefix, k)) + } + return results, nil +} + +func (s *storage) Stat(key string) (certmagic.KeyInfo, error) { + records, err := s.store.Read(key) + if err != nil { + return certmagic.KeyInfo{}, err + } + if len(records) != 1 { + return certmagic.KeyInfo{}, fmt.Errorf("ACME Storage: multiple records matched key %s", key) + } + b := bytes.NewBuffer(records[0].Value) + d := gob.NewDecoder(b) + var f file + err = d.Decode(&f) + if err != nil { + return certmagic.KeyInfo{}, err + } + return certmagic.KeyInfo{ + Key: key, + Modified: f.lastModified, + Size: int64(len(f.contents)), + IsTerminal: false, + }, nil +} diff --git a/api/server/acme/options.go b/api/server/acme/options.go new file mode 100644 index 00000000..cd7f55fd --- /dev/null +++ b/api/server/acme/options.go @@ -0,0 +1,73 @@ +package acme + +import "github.com/go-acme/lego/v3/challenge" + +// Option (or Options) are passed to New() to configure providers +type Option func(o *Options) + +// Options represents various options you can present to ACME providers +type Options struct { + // AcceptTLS must be set to true to indicate that you have read your + // provider's terms of service. + AcceptToS bool + // CA is the CA to use + CA string + // ChallengeProvider is a go-acme/lego challenge provider. Set this if you + // want to use DNS Challenges. Otherwise, tls-alpn-01 will be used + ChallengeProvider challenge.Provider + // Issue certificates for domains on demand. Otherwise, certs will be + // retrieved / issued on start-up. + OnDemand bool + // Cache is a storage interface. Most ACME libraries have an cache, but + // there's no defined interface, so if you consume this option + // sanity check it before using. + Cache interface{} +} + +// AcceptToS indicates whether you accept your CA's terms of service +func AcceptToS(b bool) Option { + return func(o *Options) { + o.AcceptToS = b + } +} + +// CA sets the CA of an acme.Options +func CA(CA string) Option { + return func(o *Options) { + o.CA = CA + } +} + +// ChallengeProvider sets the Challenge provider of an acme.Options +// if set, it enables the DNS challenge, otherwise tls-alpn-01 will be used. +func ChallengeProvider(p challenge.Provider) Option { + return func(o *Options) { + o.ChallengeProvider = p + } +} + +// OnDemand enables on-demand certificate issuance. Not recommended for use +// with the DNS challenge, as the first connection may be very slow. +func OnDemand(b bool) Option { + return func(o *Options) { + o.OnDemand = b + } +} + +// Cache provides a cache / storage interface to the underlying ACME library +// as there is no standard, this needs to be validated by the underlying +// implentation. +func Cache(c interface{}) Option { + return func(o *Options) { + o.Cache = c + } +} + +// Default uses the Let's Encrypt Production CA, with DNS Challenge disabled. +func Default() []Option { + return []Option{ + AcceptToS(true), + CA(LetsEncryptProductionCA), + OnDemand(true), + } +} From 7d2afa34a018989621ca44de62882b838f46bc86 Mon Sep 17 00:00:00 2001 From: Jake Sanders Date: Wed, 16 Oct 2019 12:58:14 +0100 Subject: [PATCH 2/2] Implementation and tests for certmagic.Storage interface --- api/server/acme/certmagic/certmagic_test.go | 140 +++++++++++++++++++- api/server/acme/certmagic/storage.go | 26 ++-- 2 files changed, 149 insertions(+), 17 deletions(-) diff --git a/api/server/acme/certmagic/certmagic_test.go b/api/server/acme/certmagic/certmagic_test.go index 52ede19c..367f6360 100644 --- a/api/server/acme/certmagic/certmagic_test.go +++ b/api/server/acme/certmagic/certmagic_test.go @@ -2,11 +2,16 @@ package certmagic import ( "os" + "reflect" + "sort" "testing" + "time" "github.com/go-acme/lego/v3/providers/dns/cloudflare" "github.com/mholt/certmagic" "github.com/micro/go-micro/api/server/acme" + "github.com/micro/go-micro/config/options" + cloudflarestorage "github.com/micro/go-micro/store/cloudflare" "github.com/micro/go-micro/sync/lock/memory" ) @@ -43,13 +48,140 @@ func TestCertMagic(t *testing.T) { } func TestStorageImplementation(t *testing.T) { - var s certmagic.Storage - s = &storage{ - lock: memory.NewLock(), + apiToken, accountID := os.Getenv("CF_API_TOKEN"), os.Getenv("CF_ACCOUNT_ID") + kvID := os.Getenv("KV_NAMESPACE_ID") + if len(apiToken) == 0 || len(accountID) == 0 || len(kvID) == 0 { + t.Skip("No Cloudflare API keys available, skipping test") } + + var s certmagic.Storage + st, err := cloudflarestorage.New( + options.WithValue("CF_API_TOKEN", apiToken), + options.WithValue("CF_ACCOUNT_ID", accountID), + options.WithValue("KV_NAMESPACE_ID", kvID), + ) + if err != nil { + t.Fatalf("Couldn't initialise cloudflare storage: %s\n", err.Error()) + } + s = &storage{ + lock: memory.NewLock(), + store: st, + } + + // Test Lock if err := s.Lock("test"); err != nil { t.Error(err) } - s.Unlock("test") + + // Test Unlock + if err := s.Unlock("test"); err != nil { + t.Error(err) + } + + // Test data + testdata := []struct { + key string + value []byte + }{ + {key: "/foo/a", value: []byte("lorem")}, + {key: "/foo/b", value: []byte("ipsum")}, + {key: "/foo/c", value: []byte("dolor")}, + {key: "/foo/d", value: []byte("sit")}, + {key: "/bar/a", value: []byte("amet")}, + {key: "/bar/b", value: []byte("consectetur")}, + {key: "/bar/c", value: []byte("adipiscing")}, + {key: "/bar/d", value: []byte("elit")}, + {key: "/foo/bar/a", value: []byte("sed")}, + {key: "/foo/bar/b", value: []byte("do")}, + {key: "/foo/bar/c", value: []byte("eiusmod")}, + {key: "/foo/bar/d", value: []byte("tempor")}, + {key: "/foo/bar/baz/a", value: []byte("incididunt")}, + {key: "/foo/bar/baz/b", value: []byte("ut")}, + {key: "/foo/bar/baz/c", value: []byte("labore")}, + {key: "/foo/bar/baz/d", value: []byte("et")}, + // a duplicate just in case there's any edge cases + {key: "/foo/a", value: []byte("lorem")}, + } + + // Test Store + for _, d := range testdata { + if err := s.Store(d.key, d.value); err != nil { + t.Error(err.Error()) + } + } + + // Test Load + for _, d := range testdata { + if value, err := s.Load(d.key); err != nil { + t.Error(err.Error()) + } else { + if !reflect.DeepEqual(value, d.value) { + t.Errorf("Load %s: expected %v, got %v", d.key, d.value, value) + } + } + } + + // Test Exists + for _, d := range testdata { + if !s.Exists(d.key) { + t.Errorf("%s should exist, but doesn't\n", d.key) + } + } + + // Test List + if list, err := s.List("/", true); err != nil { + t.Error(err.Error()) + } else { + var expected []string + for i, d := range testdata { + if i != len(testdata)-1 { + // Don't store the intentionally duplicated key + expected = append(expected, d.key) + } + } + sort.Strings(expected) + sort.Strings(list) + if !reflect.DeepEqual(expected, list) { + t.Errorf("List: Expected %v, got %v\n", expected, list) + } + } + if list, err := s.List("/foo", false); err != nil { + t.Error(err.Error()) + } else { + sort.Strings(list) + expected := []string{"/foo/a", "/foo/b", "/foo/bar", "/foo/c", "/foo/d"} + if !reflect.DeepEqual(expected, list) { + t.Errorf("List: expected %s, got %s\n", expected, list) + } + } + + // Test Stat + for _, d := range testdata { + info, err := s.Stat(d.key) + if err != nil { + t.Error(err.Error()) + } else { + if info.Key != d.key { + t.Errorf("Stat().Key: expected %s, got %s\n", d.key, info.Key) + } + if info.Size != int64(len(d.value)) { + t.Errorf("Stat().Size: expected %d, got %d\n", len(d.value), info.Size) + } + if time.Since(info.Modified) > time.Minute { + t.Errorf("Stat().Modified: expected time since last modified to be < 1 minute, got %v\n", time.Since(info.Modified)) + } + } + + } + + // Test Delete + for _, d := range testdata { + if err := s.Delete(d.key); err != nil { + t.Error(err.Error()) + } + } + + // New interface doesn't return an error, so call it in case any log.Fatal + // happens New(acme.Cache(s)) } diff --git a/api/server/acme/certmagic/storage.go b/api/server/acme/certmagic/storage.go index b1bb92b4..d0dd3ab9 100644 --- a/api/server/acme/certmagic/storage.go +++ b/api/server/acme/certmagic/storage.go @@ -13,12 +13,12 @@ import ( "github.com/micro/go-micro/sync/lock" ) -// file represents a "file" that will be stored in store.Store - the contents and last modified time -type file struct { +// File represents a "File" that will be stored in store.Store - the contents and last modified time +type File struct { // last modified time - lastModified time.Time - // contents - contents []byte + LastModified time.Time + // Contents + Contents []byte } // storage is an implementation of certmagic.Storage using micro's sync.Map and store.Store interfaces. @@ -38,9 +38,9 @@ func (s *storage) Unlock(key string) error { } func (s *storage) Store(key string, value []byte) error { - f := file{ - lastModified: time.Now(), - contents: value, + f := File{ + LastModified: time.Now(), + Contents: value, } buf := &bytes.Buffer{} e := gob.NewEncoder(buf) @@ -64,12 +64,12 @@ func (s *storage) Load(key string) ([]byte, error) { } b := bytes.NewBuffer(records[0].Value) d := gob.NewDecoder(b) - var f file + var f File err = d.Decode(&f) if err != nil { return nil, err } - return f.contents, nil + return f.Contents, nil } func (s *storage) Delete(key string) error { @@ -120,15 +120,15 @@ func (s *storage) Stat(key string) (certmagic.KeyInfo, error) { } b := bytes.NewBuffer(records[0].Value) d := gob.NewDecoder(b) - var f file + var f File err = d.Decode(&f) if err != nil { return certmagic.KeyInfo{}, err } return certmagic.KeyInfo{ Key: key, - Modified: f.lastModified, - Size: int64(len(f.contents)), + Modified: f.LastModified, + Size: int64(len(f.Contents)), IsTerminal: false, }, nil }