coreos-cloudinit: bump to 0.9.3

configdrive: Remove broken support for ec2 metadata

coreos-cloudinit.go

@@ -8,13 +8,18 @@ import (
 	"time"
 
 	"github.com/coreos/coreos-cloudinit/datasource"
+	"github.com/coreos/coreos-cloudinit/datasource/configdrive"
+	"github.com/coreos/coreos-cloudinit/datasource/file"
+	"github.com/coreos/coreos-cloudinit/datasource/metadata/ec2"
+	"github.com/coreos/coreos-cloudinit/datasource/proc_cmdline"
+	"github.com/coreos/coreos-cloudinit/datasource/url"
 	"github.com/coreos/coreos-cloudinit/initialize"
 	"github.com/coreos/coreos-cloudinit/pkg"
 	"github.com/coreos/coreos-cloudinit/system"
 )
 
 const (
-	version               = "0.9.1"
+	version               = "0.9.3"
 	datasourceInterval    = 100 * time.Millisecond
 	datasourceMaxInterval = 30 * time.Second
 	datasourceTimeout     = 5 * time.Minute
@@ -24,11 +29,12 @@ var (
 	printVersion  bool
 	ignoreFailure bool
 	sources       struct {
-		file            string
+		file               string
-		configDrive     string
+		configDrive        string
-		metadataService bool
+		metadataService    bool
-		url             string
+		ec2MetadataService string
-		procCmdLine     bool
+		url                string
+		procCmdLine        bool
 	}
 	convertNetconf string
 	workspace      string
@@ -40,9 +46,10 @@ func init() {
 	flag.BoolVar(&ignoreFailure, "ignore-failure", false, "Exits with 0 status in the event of malformed input from user-data")
 	flag.StringVar(&sources.file, "from-file", "", "Read user-data from provided file")
 	flag.StringVar(&sources.configDrive, "from-configdrive", "", "Read data from provided cloud-drive directory")
-	flag.BoolVar(&sources.metadataService, "from-metadata-service", false, "Download data from metadata service")
+	flag.BoolVar(&sources.metadataService, "from-metadata-service", false, "[DEPRECATED - Use -from-ec2-metadata] Download data from metadata service")
+	flag.StringVar(&sources.ec2MetadataService, "from-ec2-metadata", "", "Download data from the provided metadata service")
 	flag.StringVar(&sources.url, "from-url", "", "Download user-data from provided url")
-	flag.BoolVar(&sources.procCmdLine, "from-proc-cmdline", false, fmt.Sprintf("Parse %s for '%s=<url>', using the cloud-config served by an HTTP GET to <url>", datasource.ProcCmdlineLocation, datasource.ProcCmdlineCloudConfigFlag))
+	flag.BoolVar(&sources.procCmdLine, "from-proc-cmdline", false, fmt.Sprintf("Parse %s for '%s=<url>', using the cloud-config served by an HTTP GET to <url>", proc_cmdline.ProcCmdlineLocation, proc_cmdline.ProcCmdlineCloudConfigFlag))
 	flag.StringVar(&convertNetconf, "convert-netconf", "", "Read the network config provided in cloud-drive and translate it from the specified format into networkd unit files (requires the -from-configdrive flag)")
 	flag.StringVar(&workspace, "workspace", "/var/lib/coreos-cloudinit", "Base directory coreos-cloudinit should use to store data")
 	flag.StringVar(&sshKeyName, "ssh-key-name", initialize.DefaultSSHKeyName, "Add SSH keys to the system with the given name")
@@ -78,7 +85,7 @@ func main() {
 
 	dss := getDatasources()
 	if len(dss) == 0 {
-		fmt.Println("Provide at least one of --from-file, --from-configdrive, --from-metadata-service, --from-url or --from-proc-cmdline")
+		fmt.Println("Provide at least one of --from-file, --from-configdrive, --from-ec2-metadata, --from-url or --from-proc-cmdline")
 		os.Exit(1)
 	}
 
@@ -172,7 +179,7 @@ func main() {
 func mergeCloudConfig(mdcc, udcc initialize.CloudConfig) (cc initialize.CloudConfig) {
 	if mdcc.Hostname != "" {
 		if udcc.Hostname != "" {
-			fmt.Printf("Warning: user-data hostname (%s) overrides metadata hostname (%s)", udcc.Hostname, mdcc.Hostname)
+			fmt.Printf("Warning: user-data hostname (%s) overrides metadata hostname (%s)\n", udcc.Hostname, mdcc.Hostname)
 		} else {
 			udcc.Hostname = mdcc.Hostname
 		}
@@ -183,7 +190,7 @@ func mergeCloudConfig(mdcc, udcc initialize.CloudConfig) (cc initialize.CloudCon
 	}
 	if mdcc.NetworkConfigPath != "" {
 		if udcc.NetworkConfigPath != "" {
-			fmt.Printf("Warning: user-data NetworkConfigPath %s overrides metadata NetworkConfigPath %s", udcc.NetworkConfigPath, mdcc.NetworkConfigPath)
+			fmt.Printf("Warning: user-data NetworkConfigPath %s overrides metadata NetworkConfigPath %s\n", udcc.NetworkConfigPath, mdcc.NetworkConfigPath)
 		} else {
 			udcc.NetworkConfigPath = mdcc.NetworkConfigPath
 		}
@@ -196,19 +203,22 @@ func mergeCloudConfig(mdcc, udcc initialize.CloudConfig) (cc initialize.CloudCon
 func getDatasources() []datasource.Datasource {
 	dss := make([]datasource.Datasource, 0, 5)
 	if sources.file != "" {
-		dss = append(dss, datasource.NewLocalFile(sources.file))
+		dss = append(dss, file.NewDatasource(sources.file))
 	}
 	if sources.url != "" {
-		dss = append(dss, datasource.NewRemoteFile(sources.url))
+		dss = append(dss, url.NewDatasource(sources.url))
 	}
 	if sources.configDrive != "" {
-		dss = append(dss, datasource.NewConfigDrive(sources.configDrive))
+		dss = append(dss, configdrive.NewDatasource(sources.configDrive))
 	}
 	if sources.metadataService {
-		dss = append(dss, datasource.NewMetadataService())
+		dss = append(dss, ec2.NewDatasource(ec2.DefaultAddress))
+	}
+	if sources.ec2MetadataService != "" {
+		dss = append(dss, ec2.NewDatasource(sources.ec2MetadataService))
 	}
 	if sources.procCmdLine {
-		dss = append(dss, datasource.NewProcCmdline())
+		dss = append(dss, proc_cmdline.NewDatasource())
 	}
 	return dss
 }
@@ -240,7 +250,7 @@ func selectDatasource(sources []datasource.Datasource) datasource.Datasource {
 				select {
 				case <-stop:
 					return
-				case <-time.Tick(duration):
+				case <-time.After(duration):
 					duration = pkg.ExpBackoff(duration, datasourceMaxInterval)
 				}
 			}
@@ -257,7 +267,7 @@ func selectDatasource(sources []datasource.Datasource) datasource.Datasource {
 	select {
 	case s = <-ds:
 	case <-done:
-	case <-time.Tick(datasourceTimeout):
+	case <-time.After(datasourceTimeout):
 	}
 
 	close(stop)

datasource/configdrive/configdrive.go

@@ -1,17 +1,22 @@
-package datasource
+package configdrive
 
 import (
+	"fmt"
 	"io/ioutil"
 	"os"
 	"path"
 )
 
+const (
+	openstackApiVersion = "latest"
+)
+
 type configDrive struct {
 	root     string
 	readFile func(filename string) ([]byte, error)
 }
 
-func NewConfigDrive(root string) *configDrive {
+func NewDatasource(root string) *configDrive {
 	return &configDrive{root, ioutil.ReadFile}
 }
 
@@ -28,34 +33,28 @@ func (cd *configDrive) ConfigRoot() string {
 	return cd.openstackRoot()
 }
 
-// FetchMetadata attempts to retrieve metadata from ec2/2009-04-04/meta_data.json.
 func (cd *configDrive) FetchMetadata() ([]byte, error) {
-	return cd.tryReadFile(path.Join(cd.ec2Root(), "meta_data.json"))
+	return cd.tryReadFile(path.Join(cd.openstackVersionRoot(), "meta_data.json"))
 }
 
-// FetchUserdata attempts to retrieve the userdata from ec2/2009-04-04/user_data.
-// If no data is found, it will attempt to read from openstack/latest/user_data.
 func (cd *configDrive) FetchUserdata() ([]byte, error) {
-	bytes, err := cd.tryReadFile(path.Join(cd.ec2Root(), "user_data"))
+	return cd.tryReadFile(path.Join(cd.openstackVersionRoot(), "user_data"))
-	if bytes == nil && err == nil {
-		bytes, err = cd.tryReadFile(path.Join(cd.openstackRoot(), "user_data"))
-	}
-	return bytes, err
 }
 
 func (cd *configDrive) Type() string {
 	return "cloud-drive"
 }
 
-func (cd *configDrive) ec2Root() string {
+func (cd *configDrive) openstackRoot() string {
-	return path.Join(cd.root, "ec2", Ec2ApiVersion)
+	return path.Join(cd.root, "openstack")
 }
 
-func (cd *configDrive) openstackRoot() string {
+func (cd *configDrive) openstackVersionRoot() string {
-	return path.Join(cd.root, "openstack", "latest")
+	return path.Join(cd.openstackRoot(), openstackApiVersion)
 }
 
 func (cd *configDrive) tryReadFile(filename string) ([]byte, error) {
+	fmt.Printf("Attempting to read from %q\n", filename)
 	data, err := cd.readFile(filename)
 	if os.IsNotExist(err) {
 		err = nil

datasource/configdrive/configdrive_test.go

@@ -1,4 +1,4 @@
-package datasource
+package configdrive
 
 import (
 	"os"
@@ -16,7 +16,7 @@ func (m mockFilesystem) readFile(filename string) ([]byte, error) {
 	return nil, os.ErrNotExist
 }
 
-func TestCDFetchMetadata(t *testing.T) {
+func TestFetchMetadata(t *testing.T) {
 	for _, tt := range []struct {
 		root     string
 		filename string
@@ -29,13 +29,13 @@ func TestCDFetchMetadata(t *testing.T) {
 		},
 		{
 			"/",
-			"/ec2/2009-04-04/meta_data.json",
+			"/openstack/latest/meta_data.json",
-			mockFilesystem([]string{"/ec2/2009-04-04/meta_data.json"}),
+			mockFilesystem([]string{"/openstack/latest/meta_data.json"}),
 		},
 		{
 			"/media/configdrive",
-			"/media/configdrive/ec2/2009-04-04/meta_data.json",
+			"/media/configdrive/openstack/latest/meta_data.json",
-			mockFilesystem([]string{"/media/configdrive/ec2/2009-04-04/meta_data.json"}),
+			mockFilesystem([]string{"/media/configdrive/openstack/latest/meta_data.json"}),
 		},
 	} {
 		cd := configDrive{tt.root, tt.files.readFile}
@@ -49,7 +49,7 @@ func TestCDFetchMetadata(t *testing.T) {
 	}
 }
 
-func TestCDFetchUserdata(t *testing.T) {
+func TestFetchUserdata(t *testing.T) {
 	for _, tt := range []struct {
 		root     string
 		filename string
@@ -60,25 +60,15 @@ func TestCDFetchUserdata(t *testing.T) {
 			"",
 			mockFilesystem{},
 		},
-		{
-			"/",
-			"/ec2/2009-04-04/user_data",
-			mockFilesystem([]string{"/ec2/2009-04-04/user_data"}),
-		},
 		{
 			"/",
 			"/openstack/latest/user_data",
 			mockFilesystem([]string{"/openstack/latest/user_data"}),
 		},
-		{
-			"/",
-			"/ec2/2009-04-04/user_data",
-			mockFilesystem([]string{"/openstack/latest/user_data", "/ec2/2009-04-04/user_data"}),
-		},
 		{
 			"/media/configdrive",
-			"/media/configdrive/ec2/2009-04-04/user_data",
+			"/media/configdrive/openstack/latest/user_data",
-			mockFilesystem([]string{"/media/configdrive/ec2/2009-04-04/user_data"}),
+			mockFilesystem([]string{"/media/configdrive/openstack/latest/user_data"}),
 		},
 	} {
 		cd := configDrive{tt.root, tt.files.readFile}
@@ -92,18 +82,18 @@ func TestCDFetchUserdata(t *testing.T) {
 	}
 }
 
-func TestCDConfigRoot(t *testing.T) {
+func TestConfigRoot(t *testing.T) {
 	for _, tt := range []struct {
 		root       string
 		configRoot string
 	}{
 		{
 			"/",
-			"/openstack/latest",
+			"/openstack",
 		},
 		{
 			"/media/configdrive",
-			"/media/configdrive/openstack/latest",
+			"/media/configdrive/openstack",
 		},
 	} {
 		cd := configDrive{tt.root, nil}
@@ -112,3 +102,24 @@ func TestCDConfigRoot(t *testing.T) {
 		}
 	}
 }
+
+func TestNewDatasource(t *testing.T) {
+	for _, tt := range []struct {
+		root       string
+		expectRoot string
+	}{
+		{
+			root:       "",
+			expectRoot: "",
+		},
+		{
+			root:       "/media/configdrive",
+			expectRoot: "/media/configdrive",
+		},
+	} {
+		service := NewDatasource(tt.root)
+		if service.root != tt.expectRoot {
+			t.Fatalf("bad root (%q): want %q, got %q", tt.root, tt.expectRoot, service.root)
+		}
+	}
+}

datasource/file/file.go

@@ -1,4 +1,4 @@
-package datasource
+package file
 
 import (
 	"io/ioutil"
@@ -9,7 +9,7 @@ type localFile struct {
 	path string
 }
 
-func NewLocalFile(path string) *localFile {
+func NewDatasource(path string) *localFile {
 	return &localFile{path}
 }
 

datasource/metadata/ec2/metadata.go

@@ -0,0 +1,141 @@
+package ec2
+
+import (
+	"bufio"
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"strings"
+
+	"github.com/coreos/coreos-cloudinit/pkg"
+)
+
+const (
+	DefaultAddress = "http://169.254.169.254/"
+	apiVersion     = "2009-04-04"
+	userdataUrl    = apiVersion + "/user-data"
+	metadataUrl    = apiVersion + "/meta-data"
+)
+
+type metadataService struct {
+	root   string
+	client pkg.Getter
+}
+
+func NewDatasource(root string) *metadataService {
+	if !strings.HasSuffix(root, "/") {
+		root += "/"
+	}
+	return &metadataService{root, pkg.NewHttpClient()}
+}
+
+func (ms metadataService) IsAvailable() bool {
+	_, err := ms.client.Get(ms.root + apiVersion)
+	return (err == nil)
+}
+
+func (ms metadataService) AvailabilityChanges() bool {
+	return true
+}
+
+func (ms metadataService) ConfigRoot() string {
+	return ms.root
+}
+
+func (ms metadataService) FetchMetadata() ([]byte, error) {
+	attrs := make(map[string]interface{})
+	if keynames, err := fetchAttributes(ms.client, fmt.Sprintf("%s/public-keys", ms.metadataUrl())); err == nil {
+		keyIDs := make(map[string]string)
+		for _, keyname := range keynames {
+			tokens := strings.SplitN(keyname, "=", 2)
+			if len(tokens) != 2 {
+				return nil, fmt.Errorf("malformed public key: %q", keyname)
+			}
+			keyIDs[tokens[1]] = tokens[0]
+		}
+
+		keys := make(map[string]string)
+		for name, id := range keyIDs {
+			sshkey, err := fetchAttribute(ms.client, fmt.Sprintf("%s/public-keys/%s/openssh-key", ms.metadataUrl(), id))
+			if err != nil {
+				return nil, err
+			}
+			keys[name] = sshkey
+			fmt.Printf("Found SSH key for %q\n", name)
+		}
+		attrs["public_keys"] = keys
+	} else if _, ok := err.(pkg.ErrNotFound); !ok {
+		return nil, err
+	}
+
+	if hostname, err := fetchAttribute(ms.client, fmt.Sprintf("%s/hostname", ms.metadataUrl())); err == nil {
+		attrs["hostname"] = hostname
+	} else if _, ok := err.(pkg.ErrNotFound); !ok {
+		return nil, err
+	}
+
+	if localAddr, err := fetchAttribute(ms.client, fmt.Sprintf("%s/local-ipv4", ms.metadataUrl())); err == nil {
+		attrs["local-ipv4"] = localAddr
+	} else if _, ok := err.(pkg.ErrNotFound); !ok {
+		return nil, err
+	}
+
+	if publicAddr, err := fetchAttribute(ms.client, fmt.Sprintf("%s/public-ipv4", ms.metadataUrl())); err == nil {
+		attrs["public-ipv4"] = publicAddr
+	} else if _, ok := err.(pkg.ErrNotFound); !ok {
+		return nil, err
+	}
+
+	if content_path, err := fetchAttribute(ms.client, fmt.Sprintf("%s/network_config/content_path", ms.metadataUrl())); err == nil {
+		attrs["network_config"] = map[string]string{
+			"content_path": content_path,
+		}
+	} else if _, ok := err.(pkg.ErrNotFound); !ok {
+		return nil, err
+	}
+
+	return json.Marshal(attrs)
+}
+
+func (ms metadataService) FetchUserdata() ([]byte, error) {
+	if data, err := ms.client.GetRetry(ms.userdataUrl()); err == nil {
+		return data, err
+	} else if _, ok := err.(pkg.ErrNotFound); ok {
+		return []byte{}, nil
+	} else {
+		return data, err
+	}
+}
+
+func (ms metadataService) Type() string {
+	return "ec2-metadata-service"
+}
+
+func (ms metadataService) metadataUrl() string {
+	return (ms.root + metadataUrl)
+}
+
+func (ms metadataService) userdataUrl() string {
+	return (ms.root + userdataUrl)
+}
+
+func fetchAttributes(client pkg.Getter, url string) ([]string, error) {
+	resp, err := client.GetRetry(url)
+	if err != nil {
+		return nil, err
+	}
+	scanner := bufio.NewScanner(bytes.NewBuffer(resp))
+	data := make([]string, 0)
+	for scanner.Scan() {
+		data = append(data, scanner.Text())
+	}
+	return data, scanner.Err()
+}
+
+func fetchAttribute(client pkg.Getter, url string) (string, error) {
+	if attrs, err := fetchAttributes(client, url); err == nil && len(attrs) > 0 {
+		return attrs[0], nil
+	} else {
+		return "", err
+	}
+}

datasource/metadata/ec2/metadata_test.go

@@ -0,0 +1,324 @@
+package ec2
+
+import (
+	"bytes"
+	"fmt"
+	"reflect"
+	"testing"
+
+	"github.com/coreos/coreos-cloudinit/pkg"
+)
+
+type testHttpClient struct {
+	resources map[string]string
+	err       error
+}
+
+func (t *testHttpClient) GetRetry(url string) ([]byte, error) {
+	if t.err != nil {
+		return nil, t.err
+	}
+	if val, ok := t.resources[url]; ok {
+		return []byte(val), nil
+	} else {
+		return nil, pkg.ErrNotFound{fmt.Errorf("not found: %q", url)}
+	}
+}
+
+func (t *testHttpClient) Get(url string) ([]byte, error) {
+	return t.GetRetry(url)
+}
+
+func TestAvailabilityChanges(t *testing.T) {
+	want := true
+	if ac := (metadataService{}).AvailabilityChanges(); ac != want {
+		t.Fatalf("bad AvailabilityChanges: want %q, got %q", want, ac)
+	}
+}
+
+func TestType(t *testing.T) {
+	want := "ec2-metadata-service"
+	if kind := (metadataService{}).Type(); kind != want {
+		t.Fatalf("bad type: want %q, got %q", want, kind)
+	}
+}
+
+func TestIsAvailable(t *testing.T) {
+	for _, tt := range []struct {
+		root      string
+		resources map[string]string
+		expect    bool
+	}{
+		{
+			root: "/",
+			resources: map[string]string{
+				"/2009-04-04": "",
+			},
+			expect: true,
+		},
+		{
+			root:      "/",
+			resources: map[string]string{},
+			expect:    false,
+		},
+	} {
+		service := &metadataService{tt.root, &testHttpClient{tt.resources, nil}}
+		if a := service.IsAvailable(); a != tt.expect {
+			t.Fatalf("bad isAvailable (%q): want %q, got %q", tt.resources, tt.expect, a)
+		}
+	}
+}
+
+func TestFetchUserdata(t *testing.T) {
+	for _, tt := range []struct {
+		root      string
+		resources map[string]string
+		userdata  []byte
+		clientErr error
+		expectErr error
+	}{
+		{
+			root: "/",
+			resources: map[string]string{
+				"/2009-04-04/user-data": "hello",
+			},
+			userdata: []byte("hello"),
+		},
+		{
+			root:      "/",
+			clientErr: pkg.ErrNotFound{fmt.Errorf("test not found error")},
+			userdata:  []byte{},
+		},
+		{
+			root:      "/",
+			clientErr: pkg.ErrTimeout{fmt.Errorf("test timeout error")},
+			expectErr: pkg.ErrTimeout{fmt.Errorf("test timeout error")},
+		},
+	} {
+		service := &metadataService{tt.root, &testHttpClient{tt.resources, tt.clientErr}}
+		data, err := service.FetchUserdata()
+		if Error(err) != Error(tt.expectErr) {
+			t.Fatalf("bad error (%q): want %q, got %q", tt.resources, tt.expectErr, err)
+		}
+		if !bytes.Equal(data, tt.userdata) {
+			t.Fatalf("bad userdata (%q): want %q, got %q", tt.resources, tt.userdata, data)
+		}
+	}
+}
+
+func TestUrls(t *testing.T) {
+	for _, tt := range []struct {
+		root       string
+		expectRoot string
+		userdata   string
+		metadata   string
+	}{
+		{
+			root:       "/",
+			expectRoot: "/",
+			userdata:   "/2009-04-04/user-data",
+			metadata:   "/2009-04-04/meta-data",
+		},
+		{
+			root:       "http://169.254.169.254/",
+			expectRoot: "http://169.254.169.254/",
+			userdata:   "http://169.254.169.254/2009-04-04/user-data",
+			metadata:   "http://169.254.169.254/2009-04-04/meta-data",
+		},
+	} {
+		service := &metadataService{tt.root, nil}
+		if url := service.userdataUrl(); url != tt.userdata {
+			t.Fatalf("bad url (%q): want %q, got %q", tt.root, tt.userdata, url)
+		}
+		if url := service.metadataUrl(); url != tt.metadata {
+			t.Fatalf("bad url (%q): want %q, got %q", tt.root, tt.metadata, url)
+		}
+		if url := service.ConfigRoot(); url != tt.expectRoot {
+			t.Fatalf("bad url (%q): want %q, got %q", tt.root, tt.expectRoot, url)
+		}
+	}
+}
+
+func TestFetchAttributes(t *testing.T) {
+	for _, s := range []struct {
+		resources map[string]string
+		err       error
+		tests     []struct {
+			path string
+			val  []string
+		}
+	}{
+		{
+			resources: map[string]string{
+				"/":      "a\nb\nc/",
+				"/c/":    "d\ne/",
+				"/c/e/":  "f",
+				"/a":     "1",
+				"/b":     "2",
+				"/c/d":   "3",
+				"/c/e/f": "4",
+			},
+			tests: []struct {
+				path string
+				val  []string
+			}{
+				{"/", []string{"a", "b", "c/"}},
+				{"/b", []string{"2"}},
+				{"/c/d", []string{"3"}},
+				{"/c/e/", []string{"f"}},
+			},
+		},
+		{
+			err: pkg.ErrNotFound{fmt.Errorf("test error")},
+			tests: []struct {
+				path string
+				val  []string
+			}{
+				{"", nil},
+			},
+		},
+	} {
+		client := &testHttpClient{s.resources, s.err}
+		for _, tt := range s.tests {
+			attrs, err := fetchAttributes(client, tt.path)
+			if err != s.err {
+				t.Fatalf("bad error for %q (%q): want %q, got %q", tt.path, s.resources, s.err, err)
+			}
+			if !reflect.DeepEqual(attrs, tt.val) {
+				t.Fatalf("bad fetch for %q (%q): want %q, got %q", tt.path, s.resources, tt.val, attrs)
+			}
+		}
+	}
+}
+
+func TestFetchAttribute(t *testing.T) {
+	for _, s := range []struct {
+		resources map[string]string
+		err       error
+		tests     []struct {
+			path string
+			val  string
+		}
+	}{
+		{
+			resources: map[string]string{
+				"/":      "a\nb\nc/",
+				"/c/":    "d\ne/",
+				"/c/e/":  "f",
+				"/a":     "1",
+				"/b":     "2",
+				"/c/d":   "3",
+				"/c/e/f": "4",
+			},
+			tests: []struct {
+				path string
+				val  string
+			}{
+				{"/a", "1"},
+				{"/b", "2"},
+				{"/c/d", "3"},
+				{"/c/e/f", "4"},
+			},
+		},
+		{
+			err: pkg.ErrNotFound{fmt.Errorf("test error")},
+			tests: []struct {
+				path string
+				val  string
+			}{
+				{"", ""},
+			},
+		},
+	} {
+		client := &testHttpClient{s.resources, s.err}
+		for _, tt := range s.tests {
+			attr, err := fetchAttribute(client, tt.path)
+			if err != s.err {
+				t.Fatalf("bad error for %q (%q): want %q, got %q", tt.path, s.resources, s.err, err)
+			}
+			if attr != tt.val {
+				t.Fatalf("bad fetch for %q (%q): want %q, got %q", tt.path, s.resources, tt.val, attr)
+			}
+		}
+	}
+}
+
+func TestFetchMetadata(t *testing.T) {
+	for _, tt := range []struct {
+		root      string
+		resources map[string]string
+		expect    []byte
+		clientErr error
+		expectErr error
+	}{
+		{
+			root: "/",
+			resources: map[string]string{
+				"/2009-04-04/meta-data/public-keys": "bad\n",
+			},
+			expectErr: fmt.Errorf("malformed public key: \"bad\""),
+		},
+		{
+			root: "/",
+			resources: map[string]string{
+				"/2009-04-04/meta-data/hostname":                    "host",
+				"/2009-04-04/meta-data/local-ipv4":                  "1.2.3.4",
+				"/2009-04-04/meta-data/public-ipv4":                 "5.6.7.8",
+				"/2009-04-04/meta-data/public-keys":                 "0=test1\n",
+				"/2009-04-04/meta-data/public-keys/0":               "openssh-key",
+				"/2009-04-04/meta-data/public-keys/0/openssh-key":   "key",
+				"/2009-04-04/meta-data/network_config/content_path": "path",
+			},
+			expect: []byte(`{"hostname":"host","local-ipv4":"1.2.3.4","network_config":{"content_path":"path"},"public-ipv4":"5.6.7.8","public_keys":{"test1":"key"}}`),
+		},
+		{
+			clientErr: pkg.ErrTimeout{fmt.Errorf("test error")},
+			expectErr: pkg.ErrTimeout{fmt.Errorf("test error")},
+		},
+	} {
+		service := &metadataService{tt.root, &testHttpClient{tt.resources, tt.clientErr}}
+		metadata, err := service.FetchMetadata()
+		if Error(err) != Error(tt.expectErr) {
+			t.Fatalf("bad error (%q): want %q, got %q", tt.resources, tt.expectErr, err)
+		}
+		if !bytes.Equal(metadata, tt.expect) {
+			t.Fatalf("bad fetch (%q): want %q, got %q", tt.resources, tt.expect, metadata)
+		}
+	}
+}
+
+func TestNewDatasource(t *testing.T) {
+	for _, tt := range []struct {
+		root       string
+		expectRoot string
+	}{
+		{
+			root:       "",
+			expectRoot: "/",
+		},
+		{
+			root:       "/",
+			expectRoot: "/",
+		},
+		{
+			root:       "http://169.254.169.254",
+			expectRoot: "http://169.254.169.254/",
+		},
+		{
+			root:       "http://169.254.169.254/",
+			expectRoot: "http://169.254.169.254/",
+		},
+	} {
+		service := NewDatasource(tt.root)
+		if service.root != tt.expectRoot {
+			t.Fatalf("bad root (%q): want %q, got %q", tt.root, tt.expectRoot, service.root)
+		}
+	}
+}
+
+func Error(err error) string {
+	if err != nil {
+		return err.Error()
+	}
+	return ""
+}

datasource/metadata_service.go

@@ -1,153 +0,0 @@
-package datasource
-
-import (
-	"bufio"
-	"bytes"
-	"encoding/json"
-	"fmt"
-	"strings"
-
-	"github.com/coreos/coreos-cloudinit/pkg"
-)
-
-// metadataService retrieves metadata from either an OpenStack[1] (2012-08-10)
-// or EC2[2] (2009-04-04) compatible endpoint. It will first attempt to
-// directly retrieve a JSON blob from the OpenStack endpoint. If that fails
-// with a 404, it then attempts to retrieve metadata bit-by-bit from the EC2
-// endpoint, and populates that into an equivalent JSON blob. metadataService
-// also checks for userdata from EC2 and, if that fails with a 404, OpenStack.
-//
-// [1] http://docs.openstack.org/grizzly/openstack-compute/admin/content/metadata-service.html
-// [2] http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/AESDG-chapter-instancedata.html#instancedata-data-categories
-
-const (
-	BaseUrl              = "http://169.254.169.254/"
-	Ec2UserdataUrl       = BaseUrl + Ec2ApiVersion + "/user-data"
-	Ec2MetadataUrl       = BaseUrl + Ec2ApiVersion + "/meta-data"
-	OpenstackUserdataUrl = BaseUrl + "openstack/" + OpenstackApiVersion + "/user_data"
-)
-
-type metadataService struct{}
-
-type getter interface {
-	GetRetry(string) ([]byte, error)
-}
-
-func NewMetadataService() *metadataService {
-	return &metadataService{}
-}
-
-func (ms *metadataService) IsAvailable() bool {
-	client := pkg.NewHttpClient()
-	_, err := client.Get(BaseUrl)
-	return (err == nil)
-}
-
-func (ms *metadataService) AvailabilityChanges() bool {
-	return true
-}
-
-func (ms *metadataService) ConfigRoot() string {
-	return ""
-}
-
-func (ms *metadataService) FetchMetadata() ([]byte, error) {
-	return fetchMetadata(pkg.NewHttpClient())
-}
-
-func (ms *metadataService) FetchUserdata() ([]byte, error) {
-	client := pkg.NewHttpClient()
-	if data, err := client.GetRetry(Ec2UserdataUrl); err == nil {
-		return data, err
-	} else if _, ok := err.(pkg.ErrTimeout); ok {
-		return data, err
-	}
-
-	if data, err := client.GetRetry(OpenstackUserdataUrl); err == nil {
-		return data, err
-	} else if _, ok := err.(pkg.ErrNotFound); ok {
-		return []byte{}, nil
-	} else {
-		return data, err
-	}
-}
-
-func (ms *metadataService) Type() string {
-	return "metadata-service"
-}
-
-func fetchMetadata(client getter) ([]byte, error) {
-	attrs := make(map[string]interface{})
-	if keynames, err := fetchAttributes(client, fmt.Sprintf("%s/public-keys", Ec2MetadataUrl)); err == nil {
-		keyIDs := make(map[string]string)
-		for _, keyname := range keynames {
-			tokens := strings.SplitN(keyname, "=", 2)
-			if len(tokens) != 2 {
-				return nil, fmt.Errorf("malformed public key: %q\n", keyname)
-			}
-			keyIDs[tokens[1]] = tokens[0]
-		}
-
-		keys := make(map[string]string)
-		for name, id := range keyIDs {
-			sshkey, err := fetchAttribute(client, fmt.Sprintf("%s/public-keys/%s/openssh-key", Ec2MetadataUrl, id))
-			if err != nil {
-				return nil, err
-			}
-			keys[name] = sshkey
-			fmt.Printf("Found SSH key for %q\n", name)
-		}
-		attrs["public_keys"] = keys
-	} else if _, ok := err.(pkg.ErrNotFound); !ok {
-		return nil, err
-	}
-
-	if hostname, err := fetchAttribute(client, fmt.Sprintf("%s/hostname", Ec2MetadataUrl)); err == nil {
-		attrs["hostname"] = hostname
-	} else if _, ok := err.(pkg.ErrNotFound); !ok {
-		return nil, err
-	}
-
-	if localAddr, err := fetchAttribute(client, fmt.Sprintf("%s/local-ipv4", Ec2MetadataUrl)); err == nil {
-		attrs["local-ipv4"] = localAddr
-	} else if _, ok := err.(pkg.ErrNotFound); !ok {
-		return nil, err
-	}
-
-	if publicAddr, err := fetchAttribute(client, fmt.Sprintf("%s/public-ipv4", Ec2MetadataUrl)); err == nil {
-		attrs["public-ipv4"] = publicAddr
-	} else if _, ok := err.(pkg.ErrNotFound); !ok {
-		return nil, err
-	}
-
-	if content_path, err := fetchAttribute(client, fmt.Sprintf("%s/network_config/content_path", Ec2MetadataUrl)); err == nil {
-		attrs["network_config"] = map[string]string{
-			"content_path": content_path,
-		}
-	} else if _, ok := err.(pkg.ErrNotFound); !ok {
-		return nil, err
-	}
-
-	return json.Marshal(attrs)
-}
-
-func fetchAttributes(client getter, url string) ([]string, error) {
-	resp, err := client.GetRetry(url)
-	if err != nil {
-		return nil, err
-	}
-	scanner := bufio.NewScanner(bytes.NewBuffer(resp))
-	data := make([]string, 0)
-	for scanner.Scan() {
-		data = append(data, scanner.Text())
-	}
-	return data, scanner.Err()
-}
-
-func fetchAttribute(client getter, url string) (string, error) {
-	if attrs, err := fetchAttributes(client, url); err == nil && len(attrs) > 0 {
-		return attrs[0], nil
-	} else {
-		return "", err
-	}
-}

datasource/metadata_service_test.go

@@ -1,159 +0,0 @@
-package datasource
-
-import (
-	"bytes"
-	"fmt"
-	"reflect"
-	"testing"
-
-	"github.com/coreos/coreos-cloudinit/pkg"
-)
-
-type TestHttpClient struct {
-	metadata map[string]string
-	err      error
-}
-
-func (t *TestHttpClient) GetRetry(url string) ([]byte, error) {
-	if t.err != nil {
-		return nil, t.err
-	}
-	if val, ok := t.metadata[url]; ok {
-		return []byte(val), nil
-	} else {
-		return nil, pkg.ErrNotFound{fmt.Errorf("not found: %q", url)}
-	}
-}
-
-func TestMSFetchAttributes(t *testing.T) {
-	for _, s := range []struct {
-		metadata map[string]string
-		err      error
-		tests    []struct {
-			path string
-			val  []string
-		}
-	}{
-		{
-			metadata: map[string]string{
-				"/":      "a\nb\nc/",
-				"/c/":    "d\ne/",
-				"/c/e/":  "f",
-				"/a":     "1",
-				"/b":     "2",
-				"/c/d":   "3",
-				"/c/e/f": "4",
-			},
-			tests: []struct {
-				path string
-				val  []string
-			}{
-				{"/", []string{"a", "b", "c/"}},
-				{"/b", []string{"2"}},
-				{"/c/d", []string{"3"}},
-				{"/c/e/", []string{"f"}},
-			},
-		},
-		{
-			err: pkg.ErrNotFound{fmt.Errorf("test error")},
-			tests: []struct {
-				path string
-				val  []string
-			}{
-				{"", nil},
-			},
-		},
-	} {
-		client := &TestHttpClient{s.metadata, s.err}
-		for _, tt := range s.tests {
-			attrs, err := fetchAttributes(client, tt.path)
-			if err != s.err {
-				t.Fatalf("bad error for %q (%q): want %q, got %q", tt.path, s.metadata, s.err, err)
-			}
-			if !reflect.DeepEqual(attrs, tt.val) {
-				t.Fatalf("bad fetch for %q (%q): want %q, got %q", tt.path, s.metadata, tt.val, attrs)
-			}
-		}
-	}
-}
-
-func TestMSFetchAttribute(t *testing.T) {
-	for _, s := range []struct {
-		metadata map[string]string
-		err      error
-		tests    []struct {
-			path string
-			val  string
-		}
-	}{
-		{
-			metadata: map[string]string{
-				"/":      "a\nb\nc/",
-				"/c/":    "d\ne/",
-				"/c/e/":  "f",
-				"/a":     "1",
-				"/b":     "2",
-				"/c/d":   "3",
-				"/c/e/f": "4",
-			},
-			tests: []struct {
-				path string
-				val  string
-			}{
-				{"/a", "1"},
-				{"/b", "2"},
-				{"/c/d", "3"},
-				{"/c/e/f", "4"},
-			},
-		},
-		{
-			err: pkg.ErrNotFound{fmt.Errorf("test error")},
-			tests: []struct {
-				path string
-				val  string
-			}{
-				{"", ""},
-			},
-		},
-	} {
-		client := &TestHttpClient{s.metadata, s.err}
-		for _, tt := range s.tests {
-			attr, err := fetchAttribute(client, tt.path)
-			if err != s.err {
-				t.Fatalf("bad error for %q (%q): want %q, got %q", tt.path, s.metadata, s.err, err)
-			}
-			if attr != tt.val {
-				t.Fatalf("bad fetch for %q (%q): want %q, got %q", tt.path, s.metadata, tt.val, attr)
-			}
-		}
-	}
-}
-
-func TestMSFetchMetadata(t *testing.T) {
-	for _, tt := range []struct {
-		metadata map[string]string
-		err      error
-		expect   []byte
-	}{
-		{
-			metadata: map[string]string{
-				"http://169.254.169.254/2009-04-04/meta-data/hostname":                    "host",
-				"http://169.254.169.254/2009-04-04/meta-data/public-keys":                 "0=test1\n",
-				"http://169.254.169.254/2009-04-04/meta-data/public-keys/0":               "openssh-key",
-				"http://169.254.169.254/2009-04-04/meta-data/public-keys/0/openssh-key":   "key",
-				"http://169.254.169.254/2009-04-04/meta-data/network_config/content_path": "path",
-			},
-			expect: []byte(`{"hostname":"host","network_config":{"content_path":"path"},"public_keys":{"test1":"key"}}`),
-		},
-		{err: pkg.ErrTimeout{fmt.Errorf("test error")}},
-	} {
-		client := &TestHttpClient{tt.metadata, tt.err}
-		metadata, err := fetchMetadata(client)
-		if err != tt.err {
-			t.Fatalf("bad error (%q): want %q, got %q", tt.metadata, tt.err, err)
-		}
-		if !bytes.Equal(metadata, tt.expect) {
-			t.Fatalf("bad fetch (%q): want %q, got %q", tt.metadata, tt.expect, metadata)
-		}
-	}
-}

datasource/proc_cmdline/proc_cmdline.go

@@ -1,4 +1,4 @@
-package datasource
+package proc_cmdline
 
 import (
 	"errors"
@@ -18,7 +18,7 @@ type procCmdline struct {
 	Location string
 }
 
-func NewProcCmdline() *procCmdline {
+func NewDatasource() *procCmdline {
 	return &procCmdline{Location: ProcCmdlineLocation}
 }
 

datasource/proc_cmdline/proc_cmdline_test.go

@@ -1,4 +1,4 @@
-package datasource
+package proc_cmdline
 
 import (
 	"fmt"
@@ -75,7 +75,7 @@ func TestProcCmdlineAndFetchConfig(t *testing.T) {
 		t.Errorf("Test produced error: %v", err)
 	}
 
-	p := NewProcCmdline()
+	p := NewDatasource()
 	p.Location = file.Name()
 	cfg, err := p.FetchUserdata()
 	if err != nil {

datasource/url/url.go

@@ -1,4 +1,4 @@
-package datasource
+package url
 
 import "github.com/coreos/coreos-cloudinit/pkg"
 
@@ -6,7 +6,7 @@ type remoteFile struct {
 	url string
 }
 
-func NewRemoteFile(url string) *remoteFile {
+func NewDatasource(url string) *remoteFile {
 	return &remoteFile{url}
 }
 

initialize/config.go

@@ -258,7 +258,9 @@ func Apply(cfg CloudConfig, env *Environment) error {
 	}
 
 	if env.NetconfType() != "" {
-		netconfBytes, err := ioutil.ReadFile(path.Join(env.ConfigRoot(), cfg.NetworkConfigPath))
+		filename := path.Join(env.ConfigRoot(), cfg.NetworkConfigPath)
+		log.Printf("Attempting to read config from %q\n", filename)
+		netconfBytes, err := ioutil.ReadFile(filename)
 		if err != nil {
 			return err
 		}

initialize/env_test.go

@@ -66,7 +66,7 @@ func TestEnvironmentFile(t *testing.T) {
 		"$public_ipv4":  "1.2.3.4",
 		"$private_ipv4": "5.6.7.8",
 	}
-	expect := "COREOS_PUBLIC_IPV4=1.2.3.4\nCOREOS_PRIVATE_IPV4=5.6.7.8\n"
+	expect := "COREOS_PRIVATE_IPV4=5.6.7.8\nCOREOS_PUBLIC_IPV4=1.2.3.4\n"
 
 	dir, err := ioutil.TempDir(os.TempDir(), "coreos-cloudinit-")
 	if err != nil {

initialize/etcd.go

@@ -39,7 +39,7 @@ func (ee EtcdEnvironment) String() (out string) {
 // Units creates a Unit file drop-in for etcd, using any configured
 // options and adding a default MachineID if unset.
 func (ee EtcdEnvironment) Units(root string) ([]system.Unit, error) {
-	if ee == nil {
+	if len(ee) < 1 {
 		return nil, nil
 	}
 

initialize/etcd_test.go

@@ -113,8 +113,19 @@ Environment="ETCD_PEER_BIND_ADDR=127.0.0.1:7002"
 	}
 }
 
-func TestEtcdEnvironmentWrittenToDiskDefaultToMachineID(t *testing.T) {
+func TestEtcdEnvironmentEmptyNoOp(t *testing.T) {
 	ee := EtcdEnvironment{}
+	uu, err := ee.Units("")
+	if err != nil {
+		t.Fatalf("Unexpected error: %v", err)
+	}
+	if len(uu) > 0 {
+		t.Fatalf("Generated etcd units unexpectedly: %v")
+	}
+}
+
+func TestEtcdEnvironmentWrittenToDiskDefaultToMachineID(t *testing.T) {
+	ee := EtcdEnvironment{"foo": "bar"}
 	dir, err := ioutil.TempDir(os.TempDir(), "coreos-cloudinit-")
 	if err != nil {
 		t.Fatalf("Unable to create tempdir: %v", err)
@@ -152,6 +163,7 @@ func TestEtcdEnvironmentWrittenToDiskDefaultToMachineID(t *testing.T) {
 	}
 
 	expect := `[Service]
+Environment="ETCD_FOO=bar"
 Environment="ETCD_NAME=node007"
 `
 	if string(contents) != expect {

pkg/http_client.go

@@ -57,6 +57,11 @@ type HttpClient struct {
 	client *http.Client
 }
 
+type Getter interface {
+	Get(string) ([]byte, error)
+	GetRetry(string) ([]byte, error)
+}
+
 func NewHttpClient() *HttpClient {
 	hc := &HttpClient{
 		MaxBackoff: time.Second * 5,

system/env_file.go

@@ -7,6 +7,7 @@ import (
 	"os"
 	"path"
 	"regexp"
+	"sort"
 )
 
 type EnvFile struct {
@@ -24,7 +25,7 @@ var lineLexer = regexp.MustCompile(`(?m)^((?:([a-zA-Z0-9_]+)=)?.*?)\r?\n`)
 
 // mergeEnvContents: Update the existing file contents with new values,
 // preserving variable ordering and all content this code doesn't understand.
-// All new values are appended to the bottom of the old.
+// All new values are appended to the bottom of the old, sorted by key.
 func mergeEnvContents(old []byte, pending map[string]string) []byte {
 	var buf bytes.Buffer
 	var match [][]byte
@@ -44,7 +45,8 @@ func mergeEnvContents(old []byte, pending map[string]string) []byte {
 		}
 	}
 
-	for key, value := range pending {
+	for _, key := range keys(pending) {
+		value := pending[key]
 		fmt.Fprintf(&buf, "%s=%s\n", key, value)
 	}
 
@@ -87,3 +89,12 @@ func WriteEnvFile(ef *EnvFile, root string) error {
 	_, err = WriteFile(ef.File, root)
 	return err
 }
+
+// keys returns the keys of a map in sorted order
+func keys(m map[string]string) (s []string) {
+	for k, _ := range m {
+		s = append(s, k)
+	}
+	sort.Strings(s)
+	return
+}

test

@@ -13,7 +13,16 @@ COVER=${COVER:-"-cover"}
 
 source ./build
 
-declare -a TESTPKGS=(initialize system datasource pkg network)
+declare -a TESTPKGS=(initialize
+                     system
+                     datasource
+                     datasource/configdrive
+                     datasource/file
+                     datasource/metadata/ec2
+                     datasource/proc_cmdline
+                     datasource/url
+                     pkg
+                     network)
 
 if [ -z "$PKG" ]; then
 	GOFMTPATH="$TESTPKGS coreos-cloudinit.go"