Merge pull request #186 from unistack-org/structfs

util/structfs: import https://github.com/unistack-org/go-structfs

util/structfs/README.md

@@ -0,0 +1,3 @@
+# go-structfs
+
+Expose struct data via http.FileServer

util/structfs/metadata_digitalocean.go

@@ -0,0 +1,67 @@
+package structfs
+
+import (
+	"encoding/json"
+	"net/http"
+	"strings"
+	"time"
+)
+
+type DigitalOceanMetadata struct {
+	Metadata struct {
+		V1 struct {
+			DropletID  int64    `json:"droplet_id"`
+			Hostname   string   `json:"hostname"`
+			VendorData string   `json:"vendor_data"`
+			PublicKeys []string `json:"public_keys"`
+			Region     string   `json:"region"`
+			Interfaces struct {
+				Private []struct {
+					IPv4 struct {
+						Address string `json:"ip_address"`
+						Netmask string `json:"netmask"`
+						Gateway string `json:"gateway"`
+					}
+					Mac  string `json:"mac"`
+					Type string `json:"type"`
+				} `json:"private"`
+				Public []struct {
+					IPv4 struct {
+						Address string `json:"ip_address"`
+						Netmask string `json:"netmask"`
+						Gateway string `json:"gateway"`
+					} `json:"ipv4"`
+					IPv6 struct {
+						Address string `json:"ip_address"`
+						CIDR    int    `json:"cidr"`
+						Gateway string `json:"gateway"`
+					} `json:"ipv6"`
+					Mac  string `json:"mac"`
+					Type string `json:"type"`
+				} `json:"public"`
+			} `json:"interfaces"`
+			FloatingIP struct {
+				IPv4 struct {
+					Active bool `json:"active"`
+				} `json:"ipv4"`
+			} `json:"floating_ip"`
+			DNS struct {
+				Nameservers []string `json:"nameservers"`
+			} `json:"dns"`
+			Features map[string]interface{} `json:"features"`
+		} `json:"v1"`
+	} `json:"metadata"`
+}
+
+func (stfs *DigitalOceanMetadata) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	switch r.URL.Path {
+	case "/metadata/v1.json":
+		json.NewEncoder(w).Encode(stfs.Metadata.V1)
+	default:
+		fs := FileServer(stfs, "json", time.Now())
+		idx := strings.Index(r.URL.Path[1:], "/")
+		r.URL.Path = strings.Replace(r.URL.Path[idx+1:], "/metadata/v1/", "", 1)
+		r.RequestURI = r.URL.Path
+		fs.ServeHTTP(w, r)
+	}
+}

util/structfs/metadata_ec2.go

@@ -0,0 +1,30 @@
+package structfs
+
+type EC2Metadata struct {
+	Latest struct {
+		Metadata struct {
+			AMIID              int      `json:"ami-id"`
+			AMILaunchIndex     int      `json:"ami-launch-index"`
+			AMIManifestPath    string   `json:"ami-manifest-path"`
+			AncestorAMIIDs     []int    `json:"ancestor-ami-ids"`
+			BlockDeviceMapping []string `json:"block-device-mapping"`
+			InstanceID         int      `json:"instance-id"`
+			InstanceType       string   `json:"instance-type"`
+			LocalHostname      string   `json:"local-hostname"`
+			LocalIPv4          string   `json:"local-ipv4"`
+			kernelID           int      `json:"kernel-id"`
+			Placement          string   `json:"placement"`
+			AvailabilityZone   string   `json:"availability-zone"`
+			ProductCodes       string   `json:"product-codes"`
+			PublicHostname     string   `json:"public-hostname"`
+			PublicIPv4         string   `json:"public-ipv4"`
+			PublicKeys         []struct {
+				Key []string `json:"-"`
+			} `json:"public-keys"`
+			RamdiskID      int      `json:"ramdisk-id"`
+			ReservationID  int      `json:"reservation-id"`
+			SecurityGroups []string `json:"security-groups"`
+		} `json:"meta-data"`
+		Userdata string `json:"user-data"`
+	} `json:"latest"`
+}

util/structfs/structfs.go

@@ -0,0 +1,245 @@
+package structfs
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+	"os"
+	"reflect"
+	"strings"
+	"time"
+)
+
+// FileServer creates new file server from the struct iface with specific tag and specific modtime
+func FileServer(iface interface{}, tag string, modtime time.Time) http.Handler {
+	if modtime.IsZero() {
+		modtime = time.Now()
+	}
+	return &fs{iface: iface, tag: tag, modtime: modtime}
+}
+
+func (fs *fs) ServeHTTP(w http.ResponseWriter, r *http.Request) {
+	upath := r.URL.Path
+	if !strings.HasPrefix(upath, "/") {
+		upath = "/" + upath
+		r.URL.Path = upath
+	}
+	f, err := fs.Open(r.URL.Path)
+	if err != nil {
+		w.WriteHeader(http.StatusInternalServerError)
+		w.Write([]byte(err.Error()))
+		return
+	}
+	w.Header().Set("Content-Type", "application/octet-stream")
+	http.ServeContent(w, r, r.URL.Path, fs.modtime, f)
+}
+
+type fs struct {
+	iface   interface{}
+	tag     string
+	modtime time.Time
+}
+
+type file struct {
+	name    string
+	offset  int64
+	data    []byte
+	modtime time.Time
+}
+
+type fileInfo struct {
+	name    string
+	size    int64
+	modtime time.Time
+}
+
+func (fi *fileInfo) Sys() interface{} {
+	return nil
+}
+
+func (fi *fileInfo) Size() int64 {
+	return fi.size
+}
+
+func (fi *fileInfo) Name() string {
+	return fi.name
+}
+
+func (fi *fileInfo) Mode() os.FileMode {
+	if strings.HasSuffix(fi.name, "/") {
+		return os.FileMode(0755) | os.ModeDir
+	}
+	return os.FileMode(0644)
+}
+
+func (fi *fileInfo) IsDir() bool {
+	// disables additional open /index.html
+	return false
+}
+
+func (fi *fileInfo) ModTime() time.Time {
+	return fi.modtime
+}
+
+func (f *file) Close() error {
+	return nil
+}
+
+func (f *file) Read(b []byte) (int, error) {
+	var err error
+	var n int
+
+	if f.offset >= int64(len(f.data)) {
+		return 0, io.EOF
+	}
+
+	if len(f.data) > 0 {
+		n = copy(b, f.data[f.offset:])
+	}
+
+	if n < len(b) {
+		err = io.EOF
+	}
+
+	f.offset += int64(n)
+	return n, err
+}
+
+func (f *file) Readdir(count int) ([]os.FileInfo, error) {
+	return nil, nil
+}
+
+func (f *file) Seek(offset int64, whence int) (int64, error) {
+	//	log.Printf("seek %d %d %s\n", offset, whence, f.name)
+	switch whence {
+	case os.SEEK_SET:
+		f.offset = offset
+	case os.SEEK_CUR:
+		f.offset += offset
+	case os.SEEK_END:
+		f.offset = int64(len(f.data)) + offset
+	}
+	return f.offset, nil
+
+}
+
+func (f *file) Stat() (os.FileInfo, error) {
+	return &fileInfo{name: f.name, size: int64(len(f.data)), modtime: f.modtime}, nil
+}
+
+func (fs *fs) Open(path string) (http.File, error) {
+	return newFile(path, fs.iface, fs.tag, fs.modtime)
+}
+
+func newFile(name string, iface interface{}, tag string, modtime time.Time) (*file, error) {
+	var err error
+
+	f := &file{name: name, modtime: modtime}
+	f.data, err = structItem(name, iface, tag)
+	if err != nil {
+		return nil, err
+	}
+	return f, nil
+}
+
+func structItem(path string, iface interface{}, tag string) ([]byte, error) {
+	var buf []byte
+	var err error
+	var curiface interface{}
+
+	if path == "/" {
+		return getNames(iface, tag)
+	}
+
+	idx := strings.Index(path[1:], "/")
+	switch {
+	case idx > 0:
+		curiface, err = getStruct(path[1:idx+1], iface, tag)
+		if err != nil {
+			return nil, err
+		}
+		buf, err = structItem(path[idx+1:], curiface, tag)
+	case idx == 0:
+		return getNames(iface, tag)
+	case idx < 0:
+		return getValue(path[1:], iface, tag)
+	}
+
+	return buf, err
+}
+
+func getNames(iface interface{}, tag string) ([]byte, error) {
+	var lines []string
+	s := reflectValue(iface)
+	typeOf := s.Type()
+	for i := 0; i < s.NumField(); i++ {
+		value := typeOf.Field(i).Tag.Get(tag)
+		if value != "" {
+			lines = append(lines, value)
+		}
+	}
+	if len(lines) > 0 {
+		return []byte(strings.Join(lines, "\n")), nil
+	}
+	return nil, fmt.Errorf("failed to find names")
+}
+
+func getStruct(name string, iface interface{}, tag string) (interface{}, error) {
+	s := reflectValue(iface)
+	typeOf := s.Type()
+	for i := 0; i < s.NumField(); i++ {
+		if typeOf.Field(i).Tag.Get(tag) == name {
+			return s.Field(i).Interface(), nil
+		}
+	}
+	return nil, fmt.Errorf("failed to find iface %T with name %s", iface, name)
+}
+
+func getValue(name string, iface interface{}, tag string) ([]byte, error) {
+	s := reflectValue(iface)
+	typeOf := s.Type()
+	switch typeOf.Kind() {
+	case reflect.Map:
+		return []byte(fmt.Sprintf("%v", s.MapIndex(reflect.ValueOf(name)).Interface())), nil
+	default:
+		for i := 0; i < s.NumField(); i++ {
+			if typeOf.Field(i).Tag.Get(tag) != name {
+				continue
+			}
+			ifs := s.Field(i).Interface()
+			switch s.Field(i).Kind() {
+			case reflect.Slice:
+				var lines []string
+				for k := 0; k < s.Field(i).Len(); k++ {
+					lines = append(lines, fmt.Sprintf("%v", s.Field(i).Index(k)))
+				}
+				return []byte(strings.Join(lines, "\n")), nil
+			default:
+				return []byte(fmt.Sprintf("%v", ifs)), nil
+			}
+		}
+	}
+	return nil, fmt.Errorf("failed to find %s in interface %T", name, iface)
+}
+
+func hasValidType(obj interface{}, types []reflect.Kind) bool {
+	for _, t := range types {
+		if reflect.TypeOf(obj).Kind() == t {
+			return true
+		}
+	}
+
+	return false
+}
+
+func reflectValue(obj interface{}) reflect.Value {
+	var val reflect.Value
+
+	if reflect.TypeOf(obj).Kind() == reflect.Ptr {
+		val = reflect.ValueOf(obj).Elem()
+	} else {
+		val = reflect.ValueOf(obj)
+	}
+
+	return val
+}

util/structfs/structfs_test.go

@@ -0,0 +1,133 @@
+package structfs
+
+import (
+	"encoding/json"
+	"io/ioutil"
+	"net/http"
+	"reflect"
+	"testing"
+	"time"
+)
+
+var doOrig = []byte(`{
+  "droplet_id":2756294,
+  "hostname":"sample-droplet",
+  "vendor_data":"#cloud-config\ndisable_root: false\nmanage_etc_hosts: true\n\ncloud_config_modules:\n - ssh\n - set_hostname\n - [ update_etc_hosts, once-per-instance ]\n\ncloud_final_modules:\n - scripts-vendor\n - scripts-per-once\n - scripts-per-boot\n - scripts-per-instance\n - scripts-user\n",
+  "public_keys":["ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABAQCcbi6cygCUmuNlB0KqzBpHXf7CFYb3VE4pDOf/RLJ8OFDjOM+fjF83a24QktSVIpQnHYpJJT2pQMBxD+ZmnhTbKv+OjwHSHwAfkBullAojgZKzz+oN35P4Ea4J78AvMrHw0zp5MknS+WKEDCA2c6iDRCq6/hZ13Mn64f6c372JK99X29lj/B4VQpKCQyG8PUSTFkb5DXTETGbzuiVft+vM6SF+0XZH9J6dQ7b4yD3sOder+M0Q7I7CJD4VpdVD/JFa2ycOS4A4dZhjKXzabLQXdkWHvYGgNPGA5lI73TcLUAueUYqdq3RrDRfaQ5Z0PEw0mDllCzhk5dQpkmmqNi0F sammy@digitalocean.com"],
+  "region":"nyc3",
+  "interfaces":{
+    "private":[
+      {
+        "ipv4":{
+          "ip_address":"10.132.255.113",
+          "netmask":"255.255.0.0",
+          "gateway":"10.132.0.1"
+        },
+        "mac":"04:01:2a:0f:2a:02",
+        "type":"private"
+      }
+    ],
+    "public":[
+      {
+        "ipv4":{
+          "ip_address":"104.131.20.105",
+          "netmask":"255.255.192.0",
+          "gateway":"104.131.0.1"
+        },
+        "ipv6":{
+          "ip_address":"2604:A880:0800:0010:0000:0000:017D:2001",
+          "cidr":64,
+          "gateway":"2604:A880:0800:0010:0000:0000:0000:0001"
+        },
+        "mac":"04:01:2a:0f:2a:01",
+        "type":"public"}
+    ]
+  },
+  "floating_ip": {
+    "ipv4": {
+      "active": false
+    }
+  },
+  "dns":{
+    "nameservers":[
+      "2001:4860:4860::8844",
+      "2001:4860:4860::8888",
+      "8.8.8.8"
+    ]
+  },
+  "features":{
+    "dhcp_enabled": true
+  }
+}
+`)
+
+func server(t *testing.T) {
+	stfs := DigitalOceanMetadata{}
+	err := json.Unmarshal(doOrig, &stfs.Metadata.V1)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	http.Handle("/metadata/v1/", FileServer(&stfs, "json", time.Now()))
+	http.Handle("/metadata/v1.json", &stfs)
+	go func() {
+		t.Fatal(http.ListenAndServe("127.0.0.1:8080", nil))
+	}()
+	time.Sleep(2 * time.Second)
+}
+
+func get(path string) ([]byte, error) {
+	res, err := http.Get(path)
+	if err != nil {
+		return nil, err
+	}
+	defer res.Body.Close()
+	return ioutil.ReadAll(res.Body)
+}
+
+func TestAll(t *testing.T) {
+	server(t)
+
+	var tests = []struct {
+		in  string
+		out string
+	}{
+		{"http://127.0.0.1:8080/metadata/v1/", "droplet_id\nhostname\nvendor_data\npublic_keys\nregion\ninterfaces\nfloating_ip\ndns\nfeatures"},
+		{"http://127.0.0.1:8080/metadata/v1/droplet_id", "2756294"},
+		{"http://127.0.0.1:8080/metadata/v1/dns/", "nameservers"},
+		{"http://127.0.0.1:8080/metadata/v1/dns/nameservers", "2001:4860:4860::8844\n2001:4860:4860::8888\n8.8.8.8"},
+		{"http://127.0.0.1:8080/metadata/v1/features/dhcp_enabled", "true"},
+	}
+
+	for _, tt := range tests {
+		buf, err := get(tt.in)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if string(buf) != tt.out {
+			t.Errorf("req %s output %s not match requested %s", tt.in, string(buf), tt.out)
+		}
+	}
+
+	doTest, err := get("http://127.0.0.1:8080/metadata/v1.json")
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	oSt := DigitalOceanMetadata{}
+	err = json.Unmarshal(doOrig, &oSt.Metadata.V1)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	nSt := DigitalOceanMetadata{}
+
+	err = json.Unmarshal(doTest, &nSt.Metadata.V1)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	if !reflect.DeepEqual(oSt, nSt) {
+		t.Fatalf("%v not match %v", oSt, nSt)
+	}
+}