add create and delete namespace to runtime (#1965)

* add create and delete namespace to runtime

* dial down aggressive expiry

* add logging

* fix deletenamespace

* add start of k8s unit tests

* fix workflow

* turn on k8s tests

* ease tight tests

* mkdir in workflow

* dammit -p

* setup folder

.github/workflows/tests.yml

@@ -14,6 +14,11 @@ jobs:
         go-version: 1.13
       id: go
 
+    - name: Setup Kind
+      uses: engineerd/setup-kind@v0.4.0
+      with:
+        version: v0.8.1
+
     - name: Check out code into the Go module directory
       uses: actions/checkout@v2
 
@@ -26,9 +31,12 @@ jobs:
       env:
         IN_TRAVIS_CI: yes
       run: |
+        kubectl apply -f runtime/kubernetes/test/test.yaml
+        sudo mkdir -p /var/run/secrets/kubernetes.io/serviceaccount
+        sudo chmod 777 /var/run/secrets/kubernetes.io/serviceaccount
         wget -qO- https://binaries.cockroachdb.com/cockroach-v20.1.4.linux-amd64.tgz | tar  xvz
         cockroach-v20.1.4.linux-amd64/cockroach start-single-node --insecure &
-        go test -v ./...
+        go test -tags kubernetes -v ./...
   
     - name: Notify of test failure
       if: failure()

runtime/kubernetes/kubernetes.go

@@ -435,7 +435,7 @@ func (k *kubernetes) Create(s *runtime.Service, opts ...runtime.CreateOption) er
 		if exist, err := k.namespaceExists(namespace); err == nil && !exist {
 			if err := k.createNamespace(namespace); err != nil {
 				if logger.V(logger.WarnLevel, logger.DefaultLogger) {
-					logger.Warnf("Error creating namespacr %v: %v", namespace, err)
+					logger.Warnf("Error creating namespace %v: %v", namespace, err)
 				}
 				return err
 			}
@@ -712,3 +712,35 @@ func credentialsName(service *runtime.Service) string {
 	name := fmt.Sprintf("%v-%v-credentials", service.Name, service.Version)
 	return client.SerializeResourceName(name)
 }
+
+func (k *kubernetes) CreateNamespace(ns string) error {
+	err := k.client.Create(&client.Resource{
+		Kind: "namespace",
+		Value: client.Namespace{
+			Metadata: &client.Metadata{
+				Name: ns,
+			},
+		},
+	})
+	if err != nil {
+		if logger.V(logger.ErrorLevel, logger.DefaultLogger) {
+			logger.Errorf("Error creating namespace %v: %v", ns, err)
+		}
+	}
+	return err
+}
+
+func (k *kubernetes) DeleteNamespace(ns string) error {
+	err := k.client.Delete(&client.Resource{
+		Kind: "namespace",
+		Name: ns,
+	})
+	if err != nil {
+		if err != nil {
+			if logger.V(logger.ErrorLevel, logger.DefaultLogger) {
+				logger.Errorf("Error deleting namespace %v: %v", ns, err)
+			}
+		}
+	}
+	return err
+}

runtime/kubernetes/kubernetes_test.go

@@ -0,0 +1,81 @@
+// +build kubernetes
+
+package kubernetes
+
+import (
+	"encoding/base64"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"regexp"
+	"strings"
+	"testing"
+)
+
+func setupClient(t *testing.T) {
+	files := []string{"token", "ca.crt"}
+	for _, f := range files {
+		cmd := exec.Command("kubectl", "get", "secrets", "-o",
+			fmt.Sprintf(`jsonpath="{.items[?(@.metadata.annotations['kubernetes\.io/service-account\.name']=='micro-runtime')].data.%s}"`,
+				strings.ReplaceAll(f, ".", "\\.")))
+		if outp, err := cmd.Output(); err != nil {
+			t.Fatalf("Failed to set k8s token %s", err)
+		} else {
+			outq := outp[1 : len(outp)-1]
+			decoded, err := base64.StdEncoding.DecodeString(string(outq))
+			if err != nil {
+				t.Fatalf("Failed to set k8s token %s '%s'", err, outq)
+			}
+			if err := ioutil.WriteFile("/var/run/secrets/kubernetes.io/serviceaccount/"+f, decoded, 0755); err != nil {
+				t.Fatalf("Error setting up k8s %s", err)
+			}
+		}
+
+	}
+	outp, err := exec.Command("kubectl", "config", "view", "-o", `jsonpath='{.clusters[?(@.name=="kind-kind")].cluster.server}'`).Output()
+	if err != nil {
+		t.Fatalf("Cannot find server for kind %s", err)
+	}
+	serverHost := string(outp)
+
+	split := strings.Split(serverHost[9:len(serverHost)-1], ":")
+	os.Setenv("KUBERNETES_SERVICE_HOST", split[0])
+	os.Setenv("KUBERNETES_SERVICE_PORT", split[1])
+
+}
+
+func TestNamespaceCreateDelete(t *testing.T) {
+	defer func() {
+		exec.Command("kubectl", "delete", "namespace", "foobar").Run()
+	}()
+	setupClient(t)
+	r := NewRuntime()
+	if err := r.CreateNamespace("foobar"); err != nil {
+		t.Fatalf("Unexpected error creating namespace %s", err)
+	}
+
+	if !namespaceExists(t, "foobar") {
+		t.Fatalf("Namespace foobar not found")
+	}
+	if err := r.DeleteNamespace("foobar"); err != nil {
+		t.Fatalf("Unexpected error deleting namespace %s", err)
+	}
+	if namespaceExists(t, "foobar") {
+		t.Fatalf("Namespace foobar still exists")
+	}
+}
+
+func namespaceExists(t *testing.T, ns string) bool {
+	cmd := exec.Command("kubectl", "get", "namespaces")
+	outp, err := cmd.Output()
+	if err != nil {
+		t.Fatalf("Unexpected error listing namespaces %s", err)
+	}
+	exists, err := regexp.Match(ns+"\\s+Active", outp)
+	if err != nil {
+		t.Fatalf("Error listing namespaces %s", err)
+	}
+	return exists
+
+}

runtime/kubernetes/test/test.yaml

@@ -0,0 +1,72 @@
+apiVersion: v1
+kind: ServiceAccount
+metadata:
+  name: micro-runtime
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRole
+metadata:
+  name: micro-runtime
+rules:
+  - apiGroups:
+      - ""
+    resources:
+      - pods
+      - pods/log
+      - services
+      - secrets
+      - namespaces
+    verbs:
+      - get
+      - create
+      - update
+      - delete
+      - list
+      - patch
+      - watch
+  - apiGroups:
+      - "apps"
+    resources:
+      - deployments
+    verbs:
+      - create
+      - update
+      - delete
+      - list
+      - patch
+      - watch
+  - apiGroups:
+      - ""
+    resources:
+      - secrets
+      - pods
+      - pods/logs
+    verbs:
+      - get
+      - watch
+      - list
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: ClusterRoleBinding
+metadata:
+  name: micro-runtime
+subjects:
+  - kind: ServiceAccount
+    name: micro-runtime
+    namespace: default
+roleRef:
+  kind: ClusterRole
+  name: micro-runtime
+  apiGroup: rbac.authorization.k8s.io
+---
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  name: micro-runtime
+roleRef:
+  apiGroup: rbac.authorization.k8s.io
+  kind: ClusterRole
+  name: micro-runtime
+subjects:
+  - kind: ServiceAccount
+    name: micro-runtime

runtime/local/local.go

@@ -678,3 +678,13 @@ func Entrypoint(dir string) (string, error) {
 		return "", errors.New("More than one entrypoint found")
 	}
 }
+
+func (r *localRuntime) CreateNamespace(ns string) error {
+	// noop
+	return nil
+}
+
+func (r *localRuntime) DeleteNamespace(ns string) error {
+	// noop
+	return nil
+}

runtime/runtime.go

@@ -30,6 +30,10 @@ type Runtime interface {
 	Stop() error
 	// String describes runtime
 	String() string
+	// CreateNamespace creates a new namespace in the runtime
+	CreateNamespace(string) error
+	// DeleteNamespace deletes a namespace in the runtime
+	DeleteNamespace(string) error
 }
 
 // Logs returns a log stream

store/test/store_test.go

@@ -339,7 +339,7 @@ func suffixPrefixExpiryTests(s store.Store, t *testing.T) {
 		&store.Record{
 			Key:    "foobar",
 			Value:  []byte("foobarfoobar"),
-			Expiry: time.Millisecond * 100,
+			Expiry: 1 * time.Second,
 		},
 	}
 
@@ -358,7 +358,7 @@ func suffixPrefixExpiryTests(s store.Store, t *testing.T) {
 	}
 
 	// wait for the expiry
-	time.Sleep(time.Millisecond * 200)
+	time.Sleep(1 * time.Second)
 
 	if results, err := s.Read("foo", store.ReadPrefix()); err != nil {
 		t.Errorf("Couldn't read all \"foo\" keys, got %# v (%s)", spew.Sdump(results), err)
@@ -388,12 +388,12 @@ func suffixPrefixExpiryTests(s store.Store, t *testing.T) {
 			Key:   "barfoo",
 			Value: []byte("barfoobarfoo"),
 
-			Expiry: time.Millisecond * 100,
+			Expiry: time.Second * 1,
 		},
 		&store.Record{
 			Key:    "bazbarfoo",
 			Value:  []byte("bazbarfoobazbarfoo"),
-			Expiry: 2 * time.Millisecond * 100,
+			Expiry: 2 * time.Second,
 		},
 	}
 	for _, r := range records {
@@ -410,7 +410,7 @@ func suffixPrefixExpiryTests(s store.Store, t *testing.T) {
 		}
 
 	}
-	time.Sleep(time.Millisecond * 100)
+	time.Sleep(time.Second * 1)
 	if results, err := s.Read("foo", store.ReadSuffix()); err != nil {
 		t.Errorf("Couldn't read all \"foo\" keys, got %# v (%s)", spew.Sdump(results), err)
 	} else {
@@ -420,7 +420,7 @@ func suffixPrefixExpiryTests(s store.Store, t *testing.T) {
 		}
 
 	}
-	time.Sleep(time.Millisecond * 100)
+	time.Sleep(time.Second * 1)
 	if results, err := s.Read("foo", store.ReadSuffix()); err != nil {
 		t.Errorf("Couldn't read all \"foo\" keys, got %# v (%s)", spew.Sdump(results), err)
 	} else {