Merge pull request #154 from crawford/metadata

metadata-service: Add new datasource to download from metadata service

coreos-cloudinit.go

@@ -18,6 +18,7 @@ var (
 	sources       struct {
 		file        string
 		configDrive string
+		metadataService string
 		url         string
 		procCmdLine bool
 	}
@@ -30,7 +31,8 @@ func init() {
 	flag.BoolVar(&printVersion, "version", false, "Print the version and exit")
 	flag.BoolVar(&ignoreFailure, "ignore-failure", false, "Exits with 0 status in the event of malformed input from user-data")
 	flag.StringVar(&sources.file, "from-file", "", "Read user-data from provided file")
-	flag.StringVar(&sources.configDrive, "from-configdrive", "", "Read user-data from provided cloud-drive directory")
+	flag.StringVar(&sources.configDrive, "from-configdrive", "", "Read data from provided cloud-drive directory")
+	flag.StringVar(&sources.metadataService, "from-metadata-service", "", "Download data from provided url")
 	flag.StringVar(&sources.url, "from-url", "", "Download user-data from provided url")
 	flag.BoolVar(&sources.procCmdLine, "from-proc-cmdline", false, fmt.Sprintf("Parse %s for '%s=<url>', using the cloud-config served by an HTTP GET to <url>", datasource.ProcCmdlineLocation, datasource.ProcCmdlineCloudConfigFlag))
 	flag.StringVar(&convertNetconf, "convert-netconf", "", "Read the network config provided in cloud-drive and translate it from the specified format into networkd unit files (requires the -from-configdrive flag)")
@@ -53,8 +55,8 @@ func main() {
 		os.Exit(0)
 	}
 
-	if convertNetconf != "" && sources.configDrive == "" {
+	if convertNetconf != "" && sources.configDrive == "" && sources.metadataService == "" {
-		fmt.Println("-convert-netconf flag requires -from-configdrive")
+		fmt.Println("-convert-netconf flag requires -from-configdrive or -from-metadata-service")
 		os.Exit(1)
 	}
 
@@ -68,7 +70,7 @@ func main() {
 
 	ds := getDatasource()
 	if ds == nil {
-		fmt.Println("Provide exactly one of --from-file, --from-configdrive, --from-url or --from-proc-cmdline")
+		fmt.Println("Provide exactly one of --from-file, --from-configdrive, --from-metadata-service, --from-url or --from-proc-cmdline")
 		os.Exit(1)
 	}
 
@@ -123,6 +125,10 @@ func getDatasource() datasource.Datasource {
 		ds = datasource.NewConfigDrive(sources.configDrive)
 		n++
 	}
+	if sources.metadataService != "" {
+		ds = datasource.NewMetadataService(sources.metadataService)
+		n++
+	}
 	if sources.procCmdLine {
 		ds = datasource.NewProcCmdline()
 		n++

datasource/configdrive.go

@@ -14,24 +14,24 @@ func NewConfigDrive(root string) *configDrive {
 	return &configDrive{path.Join(root, "openstack")}
 }
 
-func (self *configDrive) ConfigRoot() string {
+func (cd *configDrive) ConfigRoot() string {
-	return self.root
+	return cd.root
 }
 
-func (self *configDrive) FetchMetadata() ([]byte, error) {
+func (cd *configDrive) FetchMetadata() ([]byte, error) {
-	return self.readFile("meta_data.json")
+	return cd.readFile("meta_data.json")
 }
 
-func (self *configDrive) FetchUserdata() ([]byte, error) {
+func (cd *configDrive) FetchUserdata() ([]byte, error) {
-	return self.readFile("user_data")
+	return cd.readFile("user_data")
 }
 
-func (self *configDrive) Type() string {
+func (cd *configDrive) Type() string {
 	return "cloud-drive"
 }
 
-func (self *configDrive) readFile(filename string) ([]byte, error) {
+func (cd *configDrive) readFile(filename string) ([]byte, error) {
-	data, err := ioutil.ReadFile(path.Join(self.root, "latest", filename))
+	data, err := ioutil.ReadFile(path.Join(cd.root, "latest", filename))
 	if os.IsNotExist(err) {
 		err = nil
 	}

datasource/file.go

@@ -12,18 +12,18 @@ func NewLocalFile(path string) *localFile {
 	return &localFile{path}
 }
 
-func (self *localFile) ConfigRoot() string {
+func (f *localFile) ConfigRoot() string {
 	return ""
 }
 
-func (self *localFile) FetchMetadata() ([]byte, error) {
+func (f *localFile) FetchMetadata() ([]byte, error) {
 	return []byte{}, nil
 }
 
-func (self *localFile) FetchUserdata() ([]byte, error) {
+func (f *localFile) FetchUserdata() ([]byte, error) {
-	return ioutil.ReadFile(self.path)
+	return ioutil.ReadFile(f.path)
 }
 
-func (self *localFile) Type() string {
+func (f *localFile) Type() string {
 	return "local-file"
 }

datasource/metadata_service.go

@@ -1,28 +1,97 @@
 package datasource
 
-import "github.com/coreos/coreos-cloudinit/pkg"
+import (
+	"bufio"
+	"bytes"
+	"encoding/json"
+	"strings"
+
+	"github.com/coreos/coreos-cloudinit/pkg"
+)
 
 type metadataService struct {
 	url string
 }
 
-func NewMetadataService(url string) *metadataService {
+type getter interface {
-	return &metadataService{url}
+	Get(string) ([]byte, error)
 }
 
-func (self *metadataService) ConfigRoot() string {
+func NewMetadataService(url string) *metadataService {
+	return &metadataService{strings.TrimSuffix(url, "/")}
+}
+
+func (ms *metadataService) ConfigRoot() string {
 	return ""
 }
 
-func (self *metadataService) FetchMetadata() ([]byte, error) {
+func (ms *metadataService) FetchMetadata() ([]byte, error) {
-	return []byte{}, nil
+	client := pkg.NewHttpClient()
+	return fetchMetadata(client, ms.url)
 }
 
 func (ms *metadataService) FetchUserdata() ([]byte, error) {
 	client := pkg.NewHttpClient()
-	return client.Get(ms.url)
+	return client.Get(ms.url + "/latest/user-data")
 }
 
 func (ms *metadataService) Type() string {
 	return "metadata-service"
 }
+
+func fetchMetadata(client getter, url string) ([]byte, error) {
+	if metadata, err := client.Get(url + "/latest/meta-data.json"); err == nil {
+		return metadata, nil
+	} else if _, ok := err.(pkg.ErrTimeout); ok {
+		return nil, err
+	}
+
+	attrs, err := fetchChildAttributes(client, url+"/latest/meta-data/")
+	if err != nil {
+		return nil, err
+	}
+	return json.Marshal(attrs)
+}
+
+func fetchAttributes(client getter, url string) ([]string, error) {
+	resp, err := client.Get(url)
+	if err != nil {
+		return nil, err
+	}
+	scanner := bufio.NewScanner(bytes.NewBuffer(resp))
+	data := make([]string, 0)
+	for scanner.Scan() {
+		data = append(data, strings.Split(scanner.Text(), "=")[0])
+	}
+	return data, scanner.Err()
+}
+
+func fetchAttribute(client getter, url string) (interface{}, error) {
+	if attrs, err := fetchAttributes(client, url); err == nil {
+		return attrs[0], nil
+	} else {
+		return "", err
+	}
+}
+
+func fetchChildAttributes(client getter, url string) (interface{}, error) {
+	attrs := make(map[string]interface{})
+	attrList, err := fetchAttributes(client, url)
+	if err != nil {
+		return nil, err
+	}
+	for _, attr := range attrList {
+		var fetchFunc func(getter, string) (interface{}, error)
+		if strings.HasSuffix(attr, "/") {
+			fetchFunc = fetchChildAttributes
+		} else {
+			fetchFunc = fetchAttribute
+		}
+		if value, err := fetchFunc(client, url+attr); err == nil {
+			attrs[strings.TrimSuffix(attr, "/")] = value
+		} else {
+			return nil, err
+		}
+	}
+	return attrs, nil
+}

datasource/metadata_service_test.go

@@ -0,0 +1,168 @@
+package datasource
+
+import (
+	"bytes"
+	"fmt"
+	"reflect"
+	"testing"
+
+	"github.com/coreos/coreos-cloudinit/pkg"
+)
+
+type TestHttpClient struct {
+	metadata map[string]string
+	err      error
+}
+
+func (t *TestHttpClient) Get(url string) ([]byte, error) {
+	if t.err != nil {
+		return nil, t.err
+	}
+	if val, ok := t.metadata[url]; ok {
+		return []byte(val), nil
+	} else {
+		return nil, pkg.ErrNotFound{fmt.Errorf("not found: %q", url)}
+	}
+}
+
+func TestFetchAttributes(t *testing.T) {
+	for _, s := range []struct {
+		metadata map[string]string
+		err      error
+		tests    []struct {
+			path string
+			val  []string
+		}
+	}{
+		{
+			metadata: map[string]string{
+				"/":      "a\nb\nc/",
+				"/c/":    "d\ne/",
+				"/c/e/":  "f",
+				"/a":     "1",
+				"/b":     "2",
+				"/c/d":   "3",
+				"/c/e/f": "4",
+			},
+			tests: []struct {
+				path string
+				val  []string
+			}{
+				{"/", []string{"a", "b", "c/"}},
+				{"/b", []string{"2"}},
+				{"/c/d", []string{"3"}},
+				{"/c/e/", []string{"f"}},
+			},
+		},
+		{
+			err: pkg.ErrNotFound{fmt.Errorf("test error")},
+			tests: []struct {
+				path string
+				val  []string
+			}{
+				{"", nil},
+			},
+		},
+	} {
+		client := &TestHttpClient{s.metadata, s.err}
+		for _, tt := range s.tests {
+			attrs, err := fetchAttributes(client, tt.path)
+			if err != s.err {
+				t.Fatalf("bad error for %q (%q): want %q, got %q", tt.path, s.metadata, s.err, err)
+			}
+			if !reflect.DeepEqual(attrs, tt.val) {
+				t.Fatalf("bad fetch for %q (%q): want %q, got %q", tt.path, s.metadata, tt.val, attrs)
+			}
+		}
+	}
+}
+
+func TestFetchAttribute(t *testing.T) {
+	for _, s := range []struct {
+		metadata map[string]string
+		err      error
+		tests    []struct {
+			path string
+			val  string
+		}
+	}{
+		{
+			metadata: map[string]string{
+				"/":      "a\nb\nc/",
+				"/c/":    "d\ne/",
+				"/c/e/":  "f",
+				"/a":     "1",
+				"/b":     "2",
+				"/c/d":   "3",
+				"/c/e/f": "4",
+			},
+			tests: []struct {
+				path string
+				val  string
+			}{
+				{"/a", "1"},
+				{"/b", "2"},
+				{"/c/d", "3"},
+				{"/c/e/f", "4"},
+			},
+		},
+		{
+			err: pkg.ErrNotFound{fmt.Errorf("test error")},
+			tests: []struct {
+				path string
+				val  string
+			}{
+				{"", ""},
+			},
+		},
+	} {
+		client := &TestHttpClient{s.metadata, s.err}
+		for _, tt := range s.tests {
+			attr, err := fetchAttribute(client, tt.path)
+			if err != s.err {
+				t.Fatalf("bad error for %q (%q): want %q, got %q", tt.path, s.metadata, s.err, err)
+			}
+			if attr != tt.val {
+				t.Fatalf("bad fetch for %q (%q): want %q, got %q", tt.path, s.metadata, tt.val, attr)
+			}
+		}
+	}
+}
+
+func TestFetchMetadata(t *testing.T) {
+	for _, tt := range []struct {
+		metadata map[string]string
+		err      error
+		expect   []byte
+	}{
+		{
+			metadata: map[string]string{
+				"/latest/meta-data/":      "a\nb\nc/",
+				"/latest/meta-data/c/":    "d\ne/",
+				"/latest/meta-data/c/e/":  "f",
+				"/latest/meta-data/a":     "1",
+				"/latest/meta-data/b":     "2",
+				"/latest/meta-data/c/d":   "3",
+				"/latest/meta-data/c/e/f": "4",
+			},
+			expect: []byte(`{"a":"1","b":"2","c":{"d":"3","e":{"f":"4"}}}`),
+		},
+		{
+			metadata: map[string]string{
+				"/latest/meta-data.json": "test",
+			},
+			expect: []byte("test"),
+		},
+		{err: pkg.ErrTimeout{fmt.Errorf("test error")}},
+		{err: pkg.ErrNotFound{fmt.Errorf("test error")}},
+	} {
+		client := &TestHttpClient{tt.metadata, tt.err}
+		metadata, err := fetchMetadata(client, "")
+		if err != tt.err {
+			t.Fatalf("bad error (%q): want %q, got %q", tt.metadata, tt.err, err)
+		}
+		if !bytes.Equal(metadata, tt.expect) {
+			t.Fatalf("bad fetch (%q): want %q, got %q", tt.metadata, tt.expect, metadata)
+		}
+	}
+}

datasource/proc_cmdline.go

@@ -22,16 +22,16 @@ func NewProcCmdline() *procCmdline {
 	return &procCmdline{Location: ProcCmdlineLocation}
 }
 
-func (self *procCmdline) ConfigRoot() string {
+func (c *procCmdline) ConfigRoot() string {
 	return ""
 }
 
-func (self *procCmdline) FetchMetadata() ([]byte, error) {
+func (c *procCmdline) FetchMetadata() ([]byte, error) {
 	return []byte{}, nil
 }
 
-func (self *procCmdline) FetchUserdata() ([]byte, error) {
+func (c *procCmdline) FetchUserdata() ([]byte, error) {
-	contents, err := ioutil.ReadFile(self.Location)
+	contents, err := ioutil.ReadFile(c.Location)
 	if err != nil {
 		return nil, err
 	}
@@ -51,7 +51,7 @@ func (self *procCmdline) FetchUserdata() ([]byte, error) {
 	return cfg, nil
 }
 
-func (self *procCmdline) Type() string {
+func (c *procCmdline) Type() string {
 	return "proc-cmdline"
 }
 

datasource/url.go

@@ -0,0 +1,28 @@
+package datasource
+
+import "github.com/coreos/coreos-cloudinit/pkg"
+
+type remoteFile struct {
+	url string
+}
+
+func NewRemoteFile(url string) *remoteFile {
+	return &remoteFile{url}
+}
+
+func (f *remoteFile) ConfigRoot() string {
+	return ""
+}
+
+func (f *remoteFile) FetchMetadata() ([]byte, error) {
+	return []byte{}, nil
+}
+
+func (f *remoteFile) FetchUserdata() ([]byte, error) {
+	client := pkg.NewHttpClient()
+	return client.Get(f.url)
+}
+
+func (f *remoteFile) Type() string {
+	return "url"
+}

initialize/env.go

@@ -25,32 +25,32 @@ func NewEnvironment(root, configRoot, workspace, netconfType, sshKeyName string)
 	return &Environment{root, configRoot, workspace, netconfType, sshKeyName, substitutions}
 }
 
-func (self *Environment) Workspace() string {
+func (e *Environment) Workspace() string {
-	return path.Join(self.root, self.workspace)
+	return path.Join(e.root, e.workspace)
 }
 
-func (self *Environment) Root() string {
+func (e *Environment) Root() string {
-	return self.root
+	return e.root
 }
 
-func (self *Environment) ConfigRoot() string {
+func (e *Environment) ConfigRoot() string {
-	return self.configRoot
+	return e.configRoot
 }
 
-func (self *Environment) NetconfType() string {
+func (e *Environment) NetconfType() string {
-	return self.netconfType
+	return e.netconfType
 }
 
-func (self *Environment) SSHKeyName() string {
+func (e *Environment) SSHKeyName() string {
-	return self.sshKeyName
+	return e.sshKeyName
 }
 
-func (self *Environment) SetSSHKeyName(name string) {
+func (e *Environment) SetSSHKeyName(name string) {
-	self.sshKeyName = name
+	e.sshKeyName = name
 }
 
-func (self *Environment) Apply(data string) string {
+func (e *Environment) Apply(data string) string {
-	for key, val := range self.substitutions {
+	for key, val := range e.substitutions {
 		data = strings.Replace(data, key, val, -1)
 	}
 	return data

pkg/http_client.go

@@ -18,6 +18,20 @@ const (
 	HTTP_4xx = 4
 )
 
+type Err error
+
+type ErrTimeout struct{
+	Err
+}
+
+type ErrNotFound struct{
+	Err
+}
+
+type ErrInvalid struct{
+	Err
+}
+
 type HttpClient struct {
 	// Maximum exp backoff duration. Defaults to 5 seconds
 	MaxBackoff time.Duration
@@ -53,18 +67,18 @@ func expBackoff(interval, max time.Duration) time.Duration {
 // Fetches a given URL with support for exponential backoff and maximum retries
 func (h *HttpClient) Get(rawurl string) ([]byte, error) {
 	if rawurl == "" {
-		return nil, errors.New("URL is empty. Skipping.")
+		return nil, ErrInvalid{errors.New("URL is empty. Skipping.")}
 	}
 
 	url, err := neturl.Parse(rawurl)
 	if err != nil {
-		return nil, err
+		return nil, ErrInvalid{err}
 	}
 
 	// Unfortunately, url.Parse is too generic to throw errors if a URL does not
 	// have a valid HTTP scheme. So, we have to do this extra validation
 	if !strings.HasPrefix(url.Scheme, "http") {
-		return nil, fmt.Errorf("URL %s does not have a valid HTTP scheme. Skipping.", rawurl)
+		return nil, ErrInvalid{fmt.Errorf("URL %s does not have a valid HTTP scheme. Skipping.", rawurl)}
 	}
 
 	dataURL := url.String()
@@ -106,7 +120,7 @@ func (h *HttpClient) Get(rawurl string) ([]byte, error) {
 			}
 
 			if status == HTTP_4xx {
-				return nil, fmt.Errorf("Not found. HTTP status code: %d", resp.StatusCode)
+				return nil, ErrNotFound{fmt.Errorf("Not found. HTTP status code: %d", resp.StatusCode)}
 			}
 
 			log.Printf("Server error. HTTP status code: %d", resp.StatusCode)
@@ -119,5 +133,5 @@ func (h *HttpClient) Get(rawurl string) ([]byte, error) {
 		time.Sleep(duration)
 	}
 
-	return nil, fmt.Errorf("Unable to fetch data. Maximum retries reached: %d", h.MaxRetries)
+	return nil, ErrTimeout{fmt.Errorf("Unable to fetch data. Maximum retries reached: %d", h.MaxRetries)}
 }