metadata-service: Add metadata-service datasource

Move the old metadata-service datasource to url datasource. This new datasource
checks for the existance of meta-data.json and if it doesn't exist, walks the
meta-data directory to build a metadata blob.

coreos-cloudinit.go

@@ -18,6 +18,7 @@ var (
 	sources       struct {
 		file        string
 		configDrive string
+		metadataService string
 		url         string
 		procCmdLine bool
 	}
@@ -30,7 +31,8 @@ func init() {
 	flag.BoolVar(&printVersion, "version", false, "Print the version and exit")
 	flag.BoolVar(&ignoreFailure, "ignore-failure", false, "Exits with 0 status in the event of malformed input from user-data")
 	flag.StringVar(&sources.file, "from-file", "", "Read user-data from provided file")
-	flag.StringVar(&sources.configDrive, "from-configdrive", "", "Read user-data from provided cloud-drive directory")
+	flag.StringVar(&sources.configDrive, "from-configdrive", "", "Read data from provided cloud-drive directory")
+	flag.StringVar(&sources.metadataService, "from-metadata-service", "", "Download data from provided url")
 	flag.StringVar(&sources.url, "from-url", "", "Download user-data from provided url")
 	flag.BoolVar(&sources.procCmdLine, "from-proc-cmdline", false, fmt.Sprintf("Parse %s for '%s=<url>', using the cloud-config served by an HTTP GET to <url>", datasource.ProcCmdlineLocation, datasource.ProcCmdlineCloudConfigFlag))
 	flag.StringVar(&convertNetconf, "convert-netconf", "", "Read the network config provided in cloud-drive and translate it from the specified format into networkd unit files (requires the -from-configdrive flag)")
@@ -53,8 +55,8 @@ func main() {
 		os.Exit(0)
 	}
 
-	if convertNetconf != "" && sources.configDrive == "" {
+	if convertNetconf != "" && sources.configDrive == "" && sources.metadataService == "" {
-		fmt.Println("-convert-netconf flag requires -from-configdrive")
+		fmt.Println("-convert-netconf flag requires -from-configdrive or -from-metadata-service")
 		os.Exit(1)
 	}
 
@@ -68,7 +70,7 @@ func main() {
 
 	ds := getDatasource()
 	if ds == nil {
-		fmt.Println("Provide exactly one of --from-file, --from-configdrive, --from-url or --from-proc-cmdline")
+		fmt.Println("Provide exactly one of --from-file, --from-configdrive, --from-metadata-service, --from-url or --from-proc-cmdline")
 		os.Exit(1)
 	}
 
@@ -123,6 +125,10 @@ func getDatasource() datasource.Datasource {
 		ds = datasource.NewConfigDrive(sources.configDrive)
 		n++
 	}
+	if sources.metadataService != "" {
+		ds = datasource.NewMetadataService(sources.metadataService)
+		n++
+	}
 	if sources.procCmdLine {
 		ds = datasource.NewProcCmdline()
 		n++

datasource/configdrive.go

@@ -14,24 +14,24 @@ func NewConfigDrive(root string) *configDrive {
 	return &configDrive{path.Join(root, "openstack")}
 }
 
-func (self *configDrive) ConfigRoot() string {
+func (cd *configDrive) ConfigRoot() string {
-	return self.root
+	return cd.root
 }
 
-func (self *configDrive) FetchMetadata() ([]byte, error) {
+func (cd *configDrive) FetchMetadata() ([]byte, error) {
-	return self.readFile("meta_data.json")
+	return cd.readFile("meta_data.json")
 }
 
-func (self *configDrive) FetchUserdata() ([]byte, error) {
+func (cd *configDrive) FetchUserdata() ([]byte, error) {
-	return self.readFile("user_data")
+	return cd.readFile("user_data")
 }
 
-func (self *configDrive) Type() string {
+func (cd *configDrive) Type() string {
 	return "cloud-drive"
 }
 
-func (self *configDrive) readFile(filename string) ([]byte, error) {
+func (cd *configDrive) readFile(filename string) ([]byte, error) {
-	data, err := ioutil.ReadFile(path.Join(self.root, "latest", filename))
+	data, err := ioutil.ReadFile(path.Join(cd.root, "latest", filename))
 	if os.IsNotExist(err) {
 		err = nil
 	}

datasource/file.go

@@ -12,18 +12,18 @@ func NewLocalFile(path string) *localFile {
 	return &localFile{path}
 }
 
-func (self *localFile) ConfigRoot() string {
+func (f *localFile) ConfigRoot() string {
 	return ""
 }
 
-func (self *localFile) FetchMetadata() ([]byte, error) {
+func (f *localFile) FetchMetadata() ([]byte, error) {
 	return []byte{}, nil
 }
 
-func (self *localFile) FetchUserdata() ([]byte, error) {
+func (f *localFile) FetchUserdata() ([]byte, error) {
-	return ioutil.ReadFile(self.path)
+	return ioutil.ReadFile(f.path)
 }
 
-func (self *localFile) Type() string {
+func (f *localFile) Type() string {
 	return "local-file"
 }

datasource/metadata_service.go

@@ -1,28 +1,97 @@
 package datasource
 
-import "github.com/coreos/coreos-cloudinit/pkg"
+import (
+	"bufio"
+	"bytes"
+	"encoding/json"
+	"strings"
+
+	"github.com/coreos/coreos-cloudinit/pkg"
+)
 
 type metadataService struct {
 	url string
 }
 
-func NewMetadataService(url string) *metadataService {
+type getter interface {
-	return &metadataService{url}
+	Get(string) ([]byte, error)
 }
 
-func (self *metadataService) ConfigRoot() string {
+func NewMetadataService(url string) *metadataService {
+	return &metadataService{strings.TrimSuffix(url, "/")}
+}
+
+func (ms *metadataService) ConfigRoot() string {
 	return ""
 }
 
-func (self *metadataService) FetchMetadata() ([]byte, error) {
+func (ms *metadataService) FetchMetadata() ([]byte, error) {
-	return []byte{}, nil
+	client := pkg.NewHttpClient()
+	return fetchMetadata(client, ms.url)
 }
 
 func (ms *metadataService) FetchUserdata() ([]byte, error) {
 	client := pkg.NewHttpClient()
-	return client.Get(ms.url)
+	return client.Get(ms.url + "/latest/user-data")
 }
 
 func (ms *metadataService) Type() string {
 	return "metadata-service"
 }
+
+func fetchMetadata(client getter, url string) ([]byte, error) {
+	if metadata, err := client.Get(url + "/latest/meta-data.json"); err == nil {
+		return metadata, nil
+	} else if _, ok := err.(pkg.ErrTimeout); ok {
+		return nil, err
+	}
+
+	attrs, err := fetchChildAttributes(client, url+"/latest/meta-data/")
+	if err != nil {
+		return nil, err
+	}
+	return json.Marshal(attrs)
+}
+
+func fetchAttributes(client getter, url string) ([]string, error) {
+	resp, err := client.Get(url)
+	if err != nil {
+		return nil, err
+	}
+	scanner := bufio.NewScanner(bytes.NewBuffer(resp))
+	data := make([]string, 0)
+	for scanner.Scan() {
+		data = append(data, strings.Split(scanner.Text(), "=")[0])
+	}
+	return data, scanner.Err()
+}
+
+func fetchAttribute(client getter, url string) (interface{}, error) {
+	if attrs, err := fetchAttributes(client, url); err == nil {
+		return attrs[0], nil
+	} else {
+		return "", err
+	}
+}
+
+func fetchChildAttributes(client getter, url string) (interface{}, error) {
+	attrs := make(map[string]interface{})
+	attrList, err := fetchAttributes(client, url)
+	if err != nil {
+		return nil, err
+	}
+	for _, attr := range attrList {
+		var fetchFunc func(getter, string) (interface{}, error)
+		if strings.HasSuffix(attr, "/") {
+			fetchFunc = fetchChildAttributes
+		} else {
+			fetchFunc = fetchAttribute
+		}
+		if value, err := fetchFunc(client, url+attr); err == nil {
+			attrs[strings.TrimSuffix(attr, "/")] = value
+		} else {
+			return nil, err
+		}
+	}
+	return attrs, nil
+}

datasource/proc_cmdline.go

@@ -22,16 +22,16 @@ func NewProcCmdline() *procCmdline {
 	return &procCmdline{Location: ProcCmdlineLocation}
 }
 
-func (self *procCmdline) ConfigRoot() string {
+func (c *procCmdline) ConfigRoot() string {
 	return ""
 }
 
-func (self *procCmdline) FetchMetadata() ([]byte, error) {
+func (c *procCmdline) FetchMetadata() ([]byte, error) {
 	return []byte{}, nil
 }
 
-func (self *procCmdline) FetchUserdata() ([]byte, error) {
+func (c *procCmdline) FetchUserdata() ([]byte, error) {
-	contents, err := ioutil.ReadFile(self.Location)
+	contents, err := ioutil.ReadFile(c.Location)
 	if err != nil {
 		return nil, err
 	}
@@ -51,7 +51,7 @@ func (self *procCmdline) FetchUserdata() ([]byte, error) {
 	return cfg, nil
 }
 
-func (self *procCmdline) Type() string {
+func (c *procCmdline) Type() string {
 	return "proc-cmdline"
 }
 

datasource/url.go

@@ -0,0 +1,28 @@
+package datasource
+
+import "github.com/coreos/coreos-cloudinit/pkg"
+
+type remoteFile struct {
+	url string
+}
+
+func NewRemoteFile(url string) *remoteFile {
+	return &remoteFile{url}
+}
+
+func (f *remoteFile) ConfigRoot() string {
+	return ""
+}
+
+func (f *remoteFile) FetchMetadata() ([]byte, error) {
+	return []byte{}, nil
+}
+
+func (f *remoteFile) FetchUserdata() ([]byte, error) {
+	client := pkg.NewHttpClient()
+	return client.Get(f.url)
+}
+
+func (f *remoteFile) Type() string {
+	return "url"
+}

initialize/env.go

@@ -25,32 +25,32 @@ func NewEnvironment(root, configRoot, workspace, netconfType, sshKeyName string)
 	return &Environment{root, configRoot, workspace, netconfType, sshKeyName, substitutions}
 }
 
-func (self *Environment) Workspace() string {
+func (e *Environment) Workspace() string {
-	return path.Join(self.root, self.workspace)
+	return path.Join(e.root, e.workspace)
 }
 
-func (self *Environment) Root() string {
+func (e *Environment) Root() string {
-	return self.root
+	return e.root
 }
 
-func (self *Environment) ConfigRoot() string {
+func (e *Environment) ConfigRoot() string {
-	return self.configRoot
+	return e.configRoot
 }
 
-func (self *Environment) NetconfType() string {
+func (e *Environment) NetconfType() string {
-	return self.netconfType
+	return e.netconfType
 }
 
-func (self *Environment) SSHKeyName() string {
+func (e *Environment) SSHKeyName() string {
-	return self.sshKeyName
+	return e.sshKeyName
 }
 
-func (self *Environment) SetSSHKeyName(name string) {
+func (e *Environment) SetSSHKeyName(name string) {
-	self.sshKeyName = name
+	e.sshKeyName = name
 }
 
-func (self *Environment) Apply(data string) string {
+func (e *Environment) Apply(data string) string {
-	for key, val := range self.substitutions {
+	for key, val := range e.substitutions {
 		data = strings.Replace(data, key, val, -1)
 	}
 	return data