From 5a52b5929c7873cf5d7545008d6e93b41d608e16 Mon Sep 17 00:00:00 2001 From: Dominic Wong Date: Mon, 24 Aug 2020 16:54:39 +0100 Subject: [PATCH] add create and delete namespace to runtime (#1965) * add create and delete namespace to runtime * dial down aggressive expiry * add logging * fix deletenamespace * add start of k8s unit tests * fix workflow * turn on k8s tests * ease tight tests * mkdir in workflow * dammit -p * setup folder --- .github/workflows/tests.yml | 10 +++- runtime/kubernetes/kubernetes.go | 34 ++++++++++- runtime/kubernetes/kubernetes_test.go | 81 +++++++++++++++++++++++++++ runtime/kubernetes/test/test.yaml | 72 ++++++++++++++++++++++++ runtime/local/local.go | 10 ++++ runtime/runtime.go | 4 ++ store/test/store_test.go | 12 ++-- 7 files changed, 215 insertions(+), 8 deletions(-) create mode 100644 runtime/kubernetes/kubernetes_test.go create mode 100644 runtime/kubernetes/test/test.yaml diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index d24a5ecc..6268cf0b 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -14,6 +14,11 @@ jobs: go-version: 1.13 id: go + - name: Setup Kind + uses: engineerd/setup-kind@v0.4.0 + with: + version: v0.8.1 + - name: Check out code into the Go module directory uses: actions/checkout@v2 @@ -26,9 +31,12 @@ jobs: env: IN_TRAVIS_CI: yes run: | + kubectl apply -f runtime/kubernetes/test/test.yaml + sudo mkdir -p /var/run/secrets/kubernetes.io/serviceaccount + sudo chmod 777 /var/run/secrets/kubernetes.io/serviceaccount wget -qO- https://binaries.cockroachdb.com/cockroach-v20.1.4.linux-amd64.tgz | tar xvz cockroach-v20.1.4.linux-amd64/cockroach start-single-node --insecure & - go test -v ./... + go test -tags kubernetes -v ./... - name: Notify of test failure if: failure() diff --git a/runtime/kubernetes/kubernetes.go b/runtime/kubernetes/kubernetes.go index fcb16ca8..0eba844e 100644 --- a/runtime/kubernetes/kubernetes.go +++ b/runtime/kubernetes/kubernetes.go @@ -435,7 +435,7 @@ func (k *kubernetes) Create(s *runtime.Service, opts ...runtime.CreateOption) er if exist, err := k.namespaceExists(namespace); err == nil && !exist { if err := k.createNamespace(namespace); err != nil { if logger.V(logger.WarnLevel, logger.DefaultLogger) { - logger.Warnf("Error creating namespacr %v: %v", namespace, err) + logger.Warnf("Error creating namespace %v: %v", namespace, err) } return err } @@ -712,3 +712,35 @@ func credentialsName(service *runtime.Service) string { name := fmt.Sprintf("%v-%v-credentials", service.Name, service.Version) return client.SerializeResourceName(name) } + +func (k *kubernetes) CreateNamespace(ns string) error { + err := k.client.Create(&client.Resource{ + Kind: "namespace", + Value: client.Namespace{ + Metadata: &client.Metadata{ + Name: ns, + }, + }, + }) + if err != nil { + if logger.V(logger.ErrorLevel, logger.DefaultLogger) { + logger.Errorf("Error creating namespace %v: %v", ns, err) + } + } + return err +} + +func (k *kubernetes) DeleteNamespace(ns string) error { + err := k.client.Delete(&client.Resource{ + Kind: "namespace", + Name: ns, + }) + if err != nil { + if err != nil { + if logger.V(logger.ErrorLevel, logger.DefaultLogger) { + logger.Errorf("Error deleting namespace %v: %v", ns, err) + } + } + } + return err +} diff --git a/runtime/kubernetes/kubernetes_test.go b/runtime/kubernetes/kubernetes_test.go new file mode 100644 index 00000000..131935da --- /dev/null +++ b/runtime/kubernetes/kubernetes_test.go @@ -0,0 +1,81 @@ +// +build kubernetes + +package kubernetes + +import ( + "encoding/base64" + "fmt" + "io/ioutil" + "os" + "os/exec" + "regexp" + "strings" + "testing" +) + +func setupClient(t *testing.T) { + files := []string{"token", "ca.crt"} + for _, f := range files { + cmd := exec.Command("kubectl", "get", "secrets", "-o", + fmt.Sprintf(`jsonpath="{.items[?(@.metadata.annotations['kubernetes\.io/service-account\.name']=='micro-runtime')].data.%s}"`, + strings.ReplaceAll(f, ".", "\\."))) + if outp, err := cmd.Output(); err != nil { + t.Fatalf("Failed to set k8s token %s", err) + } else { + outq := outp[1 : len(outp)-1] + decoded, err := base64.StdEncoding.DecodeString(string(outq)) + if err != nil { + t.Fatalf("Failed to set k8s token %s '%s'", err, outq) + } + if err := ioutil.WriteFile("/var/run/secrets/kubernetes.io/serviceaccount/"+f, decoded, 0755); err != nil { + t.Fatalf("Error setting up k8s %s", err) + } + } + + } + outp, err := exec.Command("kubectl", "config", "view", "-o", `jsonpath='{.clusters[?(@.name=="kind-kind")].cluster.server}'`).Output() + if err != nil { + t.Fatalf("Cannot find server for kind %s", err) + } + serverHost := string(outp) + + split := strings.Split(serverHost[9:len(serverHost)-1], ":") + os.Setenv("KUBERNETES_SERVICE_HOST", split[0]) + os.Setenv("KUBERNETES_SERVICE_PORT", split[1]) + +} + +func TestNamespaceCreateDelete(t *testing.T) { + defer func() { + exec.Command("kubectl", "delete", "namespace", "foobar").Run() + }() + setupClient(t) + r := NewRuntime() + if err := r.CreateNamespace("foobar"); err != nil { + t.Fatalf("Unexpected error creating namespace %s", err) + } + + if !namespaceExists(t, "foobar") { + t.Fatalf("Namespace foobar not found") + } + if err := r.DeleteNamespace("foobar"); err != nil { + t.Fatalf("Unexpected error deleting namespace %s", err) + } + if namespaceExists(t, "foobar") { + t.Fatalf("Namespace foobar still exists") + } +} + +func namespaceExists(t *testing.T, ns string) bool { + cmd := exec.Command("kubectl", "get", "namespaces") + outp, err := cmd.Output() + if err != nil { + t.Fatalf("Unexpected error listing namespaces %s", err) + } + exists, err := regexp.Match(ns+"\\s+Active", outp) + if err != nil { + t.Fatalf("Error listing namespaces %s", err) + } + return exists + +} diff --git a/runtime/kubernetes/test/test.yaml b/runtime/kubernetes/test/test.yaml new file mode 100644 index 00000000..bbd2f3c6 --- /dev/null +++ b/runtime/kubernetes/test/test.yaml @@ -0,0 +1,72 @@ +apiVersion: v1 +kind: ServiceAccount +metadata: + name: micro-runtime +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: micro-runtime +rules: + - apiGroups: + - "" + resources: + - pods + - pods/log + - services + - secrets + - namespaces + verbs: + - get + - create + - update + - delete + - list + - patch + - watch + - apiGroups: + - "apps" + resources: + - deployments + verbs: + - create + - update + - delete + - list + - patch + - watch + - apiGroups: + - "" + resources: + - secrets + - pods + - pods/logs + verbs: + - get + - watch + - list +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: micro-runtime +subjects: + - kind: ServiceAccount + name: micro-runtime + namespace: default +roleRef: + kind: ClusterRole + name: micro-runtime + apiGroup: rbac.authorization.k8s.io +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: RoleBinding +metadata: + name: micro-runtime +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: micro-runtime +subjects: + - kind: ServiceAccount + name: micro-runtime \ No newline at end of file diff --git a/runtime/local/local.go b/runtime/local/local.go index 77c3d8d6..4ec0058e 100644 --- a/runtime/local/local.go +++ b/runtime/local/local.go @@ -678,3 +678,13 @@ func Entrypoint(dir string) (string, error) { return "", errors.New("More than one entrypoint found") } } + +func (r *localRuntime) CreateNamespace(ns string) error { + // noop + return nil +} + +func (r *localRuntime) DeleteNamespace(ns string) error { + // noop + return nil +} diff --git a/runtime/runtime.go b/runtime/runtime.go index ec252d02..d27d9b2d 100644 --- a/runtime/runtime.go +++ b/runtime/runtime.go @@ -30,6 +30,10 @@ type Runtime interface { Stop() error // String describes runtime String() string + // CreateNamespace creates a new namespace in the runtime + CreateNamespace(string) error + // DeleteNamespace deletes a namespace in the runtime + DeleteNamespace(string) error } // Logs returns a log stream diff --git a/store/test/store_test.go b/store/test/store_test.go index 6593f984..a8f0f0c1 100644 --- a/store/test/store_test.go +++ b/store/test/store_test.go @@ -339,7 +339,7 @@ func suffixPrefixExpiryTests(s store.Store, t *testing.T) { &store.Record{ Key: "foobar", Value: []byte("foobarfoobar"), - Expiry: time.Millisecond * 100, + Expiry: 1 * time.Second, }, } @@ -358,7 +358,7 @@ func suffixPrefixExpiryTests(s store.Store, t *testing.T) { } // wait for the expiry - time.Sleep(time.Millisecond * 200) + time.Sleep(1 * time.Second) if results, err := s.Read("foo", store.ReadPrefix()); err != nil { t.Errorf("Couldn't read all \"foo\" keys, got %# v (%s)", spew.Sdump(results), err) @@ -388,12 +388,12 @@ func suffixPrefixExpiryTests(s store.Store, t *testing.T) { Key: "barfoo", Value: []byte("barfoobarfoo"), - Expiry: time.Millisecond * 100, + Expiry: time.Second * 1, }, &store.Record{ Key: "bazbarfoo", Value: []byte("bazbarfoobazbarfoo"), - Expiry: 2 * time.Millisecond * 100, + Expiry: 2 * time.Second, }, } for _, r := range records { @@ -410,7 +410,7 @@ func suffixPrefixExpiryTests(s store.Store, t *testing.T) { } } - time.Sleep(time.Millisecond * 100) + time.Sleep(time.Second * 1) if results, err := s.Read("foo", store.ReadSuffix()); err != nil { t.Errorf("Couldn't read all \"foo\" keys, got %# v (%s)", spew.Sdump(results), err) } else { @@ -420,7 +420,7 @@ func suffixPrefixExpiryTests(s store.Store, t *testing.T) { } } - time.Sleep(time.Millisecond * 100) + time.Sleep(time.Second * 1) if results, err := s.Read("foo", store.ReadSuffix()); err != nil { t.Errorf("Couldn't read all \"foo\" keys, got %# v (%s)", spew.Sdump(results), err) } else {