Merge pull request #194 from crawford/metadata

datasource: Refactoring datasources

coreos-cloudinit.go

@@ -8,6 +8,11 @@ import (
 	"time"
 
 	"github.com/coreos/coreos-cloudinit/datasource"
+	"github.com/coreos/coreos-cloudinit/datasource/configdrive"
+	"github.com/coreos/coreos-cloudinit/datasource/file"
+	"github.com/coreos/coreos-cloudinit/datasource/metadata/ec2"
+	"github.com/coreos/coreos-cloudinit/datasource/proc_cmdline"
+	"github.com/coreos/coreos-cloudinit/datasource/url"
 	"github.com/coreos/coreos-cloudinit/initialize"
 	"github.com/coreos/coreos-cloudinit/pkg"
 	"github.com/coreos/coreos-cloudinit/system"
@@ -27,6 +32,7 @@ var (
 		file               string
 		configDrive        string
 		metadataService    bool
+		ec2MetadataService string
 		url                string
 		procCmdLine        bool
 	}
@@ -40,9 +46,10 @@ func init() {
 	flag.BoolVar(&ignoreFailure, "ignore-failure", false, "Exits with 0 status in the event of malformed input from user-data")
 	flag.StringVar(&sources.file, "from-file", "", "Read user-data from provided file")
 	flag.StringVar(&sources.configDrive, "from-configdrive", "", "Read data from provided cloud-drive directory")
-	flag.BoolVar(&sources.metadataService, "from-metadata-service", false, "Download data from metadata service")
+	flag.BoolVar(&sources.metadataService, "from-metadata-service", false, "[DEPRECATED - Use -from-ec2-metadata] Download data from metadata service")
+	flag.StringVar(&sources.ec2MetadataService, "from-ec2-metadata", "", "Download data from the provided metadata service")
 	flag.StringVar(&sources.url, "from-url", "", "Download user-data from provided url")
-	flag.BoolVar(&sources.procCmdLine, "from-proc-cmdline", false, fmt.Sprintf("Parse %s for '%s=<url>', using the cloud-config served by an HTTP GET to <url>", datasource.ProcCmdlineLocation, datasource.ProcCmdlineCloudConfigFlag))
+	flag.BoolVar(&sources.procCmdLine, "from-proc-cmdline", false, fmt.Sprintf("Parse %s for '%s=<url>', using the cloud-config served by an HTTP GET to <url>", proc_cmdline.ProcCmdlineLocation, proc_cmdline.ProcCmdlineCloudConfigFlag))
 	flag.StringVar(&convertNetconf, "convert-netconf", "", "Read the network config provided in cloud-drive and translate it from the specified format into networkd unit files (requires the -from-configdrive flag)")
 	flag.StringVar(&workspace, "workspace", "/var/lib/coreos-cloudinit", "Base directory coreos-cloudinit should use to store data")
 	flag.StringVar(&sshKeyName, "ssh-key-name", initialize.DefaultSSHKeyName, "Add SSH keys to the system with the given name")
@@ -78,7 +85,7 @@ func main() {
 
 	dss := getDatasources()
 	if len(dss) == 0 {
-		fmt.Println("Provide at least one of --from-file, --from-configdrive, --from-metadata-service, --from-url or --from-proc-cmdline")
+		fmt.Println("Provide at least one of --from-file, --from-configdrive, --from-ec2-metadata, --from-url or --from-proc-cmdline")
 		os.Exit(1)
 	}
 
@@ -172,7 +179,7 @@ func main() {
 func mergeCloudConfig(mdcc, udcc initialize.CloudConfig) (cc initialize.CloudConfig) {
 	if mdcc.Hostname != "" {
 		if udcc.Hostname != "" {
-			fmt.Printf("Warning: user-data hostname (%s) overrides metadata hostname (%s)", udcc.Hostname, mdcc.Hostname)
+			fmt.Printf("Warning: user-data hostname (%s) overrides metadata hostname (%s)\n", udcc.Hostname, mdcc.Hostname)
 		} else {
 			udcc.Hostname = mdcc.Hostname
 		}
@@ -183,7 +190,7 @@ func mergeCloudConfig(mdcc, udcc initialize.CloudConfig) (cc initialize.CloudCon
 	}
 	if mdcc.NetworkConfigPath != "" {
 		if udcc.NetworkConfigPath != "" {
-			fmt.Printf("Warning: user-data NetworkConfigPath %s overrides metadata NetworkConfigPath %s", udcc.NetworkConfigPath, mdcc.NetworkConfigPath)
+			fmt.Printf("Warning: user-data NetworkConfigPath %s overrides metadata NetworkConfigPath %s\n", udcc.NetworkConfigPath, mdcc.NetworkConfigPath)
 		} else {
 			udcc.NetworkConfigPath = mdcc.NetworkConfigPath
 		}
@@ -196,19 +203,22 @@ func mergeCloudConfig(mdcc, udcc initialize.CloudConfig) (cc initialize.CloudCon
 func getDatasources() []datasource.Datasource {
 	dss := make([]datasource.Datasource, 0, 5)
 	if sources.file != "" {
-		dss = append(dss, datasource.NewLocalFile(sources.file))
+		dss = append(dss, file.NewDatasource(sources.file))
 	}
 	if sources.url != "" {
-		dss = append(dss, datasource.NewRemoteFile(sources.url))
+		dss = append(dss, url.NewDatasource(sources.url))
 	}
 	if sources.configDrive != "" {
-		dss = append(dss, datasource.NewConfigDrive(sources.configDrive))
+		dss = append(dss, configdrive.NewDatasource(sources.configDrive))
 	}
 	if sources.metadataService {
-		dss = append(dss, datasource.NewMetadataService())
+		dss = append(dss, ec2.NewDatasource(ec2.DefaultAddress))
+	}
+	if sources.ec2MetadataService != "" {
+		dss = append(dss, ec2.NewDatasource(sources.ec2MetadataService))
 	}
 	if sources.procCmdLine {
-		dss = append(dss, datasource.NewProcCmdline())
+		dss = append(dss, proc_cmdline.NewDatasource())
 	}
 	return dss
 }
@@ -240,7 +250,7 @@ func selectDatasource(sources []datasource.Datasource) datasource.Datasource {
 				select {
 				case <-stop:
 					return
-				case <-time.Tick(duration):
+				case <-time.After(duration):
 					duration = pkg.ExpBackoff(duration, datasourceMaxInterval)
 				}
 			}
@@ -257,7 +267,7 @@ func selectDatasource(sources []datasource.Datasource) datasource.Datasource {
 	select {
 	case s = <-ds:
 	case <-done:
-	case <-time.Tick(datasourceTimeout):
+	case <-time.After(datasourceTimeout):
 	}
 
 	close(stop)

datasource/configdrive/configdrive.go

@@ -1,4 +1,4 @@
-package datasource
+package configdrive
 
 import (
 	"io/ioutil"
@@ -6,13 +6,18 @@ import (
 	"path"
 )
 
+const (
+	ec2ApiVersion       = "2009-04-04"
+	openstackApiVersion = "latest"
+)
+
 type configDrive struct {
 	root     string
 	readFile func(filename string) ([]byte, error)
 }
 
-func NewConfigDrive(root string) *configDrive {
+func NewDatasource(root string) *configDrive {
-	return &configDrive{root, ioutil.ReadFile}
+	return &configDrive{path.Join(root, "openstack"), ioutil.ReadFile}
 }
 
 func (cd *configDrive) IsAvailable() bool {
@@ -48,11 +53,11 @@ func (cd *configDrive) Type() string {
 }
 
 func (cd *configDrive) ec2Root() string {
-	return path.Join(cd.root, "ec2", Ec2ApiVersion)
+	return path.Join(cd.root, "ec2", ec2ApiVersion)
 }
 
 func (cd *configDrive) openstackRoot() string {
-	return path.Join(cd.root, "openstack", "latest")
+	return path.Join(cd.root, "openstack", openstackApiVersion)
 }
 
 func (cd *configDrive) tryReadFile(filename string) ([]byte, error) {

datasource/configdrive/configdrive_test.go

@@ -1,4 +1,4 @@
-package datasource
+package configdrive
 
 import (
 	"os"

datasource/file/file.go

@@ -1,4 +1,4 @@
-package datasource
+package file
 
 import (
 	"io/ioutil"
@@ -9,7 +9,7 @@ type localFile struct {
 	path string
 }
 
-func NewLocalFile(path string) *localFile {
+func NewDatasource(path string) *localFile {
 	return &localFile{path}
 }
 

datasource/metadata/ec2/metadata.go

@@ -0,0 +1,141 @@
+package ec2
+
+import (
+	"bufio"
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"strings"
+
+	"github.com/coreos/coreos-cloudinit/pkg"
+)
+
+const (
+	DefaultAddress = "http://169.254.169.254/"
+	apiVersion     = "2009-04-04"
+	userdataUrl    = apiVersion + "/user-data"
+	metadataUrl    = apiVersion + "/meta-data"
+)
+
+type metadataService struct {
+	root   string
+	client pkg.Getter
+}
+
+func NewDatasource(root string) *metadataService {
+	if !strings.HasSuffix(root, "/") {
+		root += "/"
+	}
+	return &metadataService{root, pkg.NewHttpClient()}
+}
+
+func (ms metadataService) IsAvailable() bool {
+	_, err := ms.client.Get(ms.root + apiVersion)
+	return (err == nil)
+}
+
+func (ms metadataService) AvailabilityChanges() bool {
+	return true
+}
+
+func (ms metadataService) ConfigRoot() string {
+	return ms.root
+}
+
+func (ms metadataService) FetchMetadata() ([]byte, error) {
+	attrs := make(map[string]interface{})
+	if keynames, err := fetchAttributes(ms.client, fmt.Sprintf("%s/public-keys", ms.metadataUrl())); err == nil {
+		keyIDs := make(map[string]string)
+		for _, keyname := range keynames {
+			tokens := strings.SplitN(keyname, "=", 2)
+			if len(tokens) != 2 {
+				return nil, fmt.Errorf("malformed public key: %q", keyname)
+			}
+			keyIDs[tokens[1]] = tokens[0]
+		}
+
+		keys := make(map[string]string)
+		for name, id := range keyIDs {
+			sshkey, err := fetchAttribute(ms.client, fmt.Sprintf("%s/public-keys/%s/openssh-key", ms.metadataUrl(), id))
+			if err != nil {
+				return nil, err
+			}
+			keys[name] = sshkey
+			fmt.Printf("Found SSH key for %q\n", name)
+		}
+		attrs["public_keys"] = keys
+	} else if _, ok := err.(pkg.ErrNotFound); !ok {
+		return nil, err
+	}
+
+	if hostname, err := fetchAttribute(ms.client, fmt.Sprintf("%s/hostname", ms.metadataUrl())); err == nil {
+		attrs["hostname"] = hostname
+	} else if _, ok := err.(pkg.ErrNotFound); !ok {
+		return nil, err
+	}
+
+	if localAddr, err := fetchAttribute(ms.client, fmt.Sprintf("%s/local-ipv4", ms.metadataUrl())); err == nil {
+		attrs["local-ipv4"] = localAddr
+	} else if _, ok := err.(pkg.ErrNotFound); !ok {
+		return nil, err
+	}
+
+	if publicAddr, err := fetchAttribute(ms.client, fmt.Sprintf("%s/public-ipv4", ms.metadataUrl())); err == nil {
+		attrs["public-ipv4"] = publicAddr
+	} else if _, ok := err.(pkg.ErrNotFound); !ok {
+		return nil, err
+	}
+
+	if content_path, err := fetchAttribute(ms.client, fmt.Sprintf("%s/network_config/content_path", ms.metadataUrl())); err == nil {
+		attrs["network_config"] = map[string]string{
+			"content_path": content_path,
+		}
+	} else if _, ok := err.(pkg.ErrNotFound); !ok {
+		return nil, err
+	}
+
+	return json.Marshal(attrs)
+}
+
+func (ms metadataService) FetchUserdata() ([]byte, error) {
+	if data, err := ms.client.GetRetry(ms.userdataUrl()); err == nil {
+		return data, err
+	} else if _, ok := err.(pkg.ErrNotFound); ok {
+		return []byte{}, nil
+	} else {
+		return data, err
+	}
+}
+
+func (ms metadataService) Type() string {
+	return "ec2-metadata-service"
+}
+
+func (ms metadataService) metadataUrl() string {
+	return (ms.root + metadataUrl)
+}
+
+func (ms metadataService) userdataUrl() string {
+	return (ms.root + userdataUrl)
+}
+
+func fetchAttributes(client pkg.Getter, url string) ([]string, error) {
+	resp, err := client.GetRetry(url)
+	if err != nil {
+		return nil, err
+	}
+	scanner := bufio.NewScanner(bytes.NewBuffer(resp))
+	data := make([]string, 0)
+	for scanner.Scan() {
+		data = append(data, scanner.Text())
+	}
+	return data, scanner.Err()
+}
+
+func fetchAttribute(client pkg.Getter, url string) (string, error) {
+	if attrs, err := fetchAttributes(client, url); err == nil && len(attrs) > 0 {
+		return attrs[0], nil
+	} else {
+		return "", err
+	}
+}

datasource/metadata/ec2/metadata_test.go

@@ -0,0 +1,324 @@
+package ec2
+
+import (
+	"bytes"
+	"fmt"
+	"reflect"
+	"testing"
+
+	"github.com/coreos/coreos-cloudinit/pkg"
+)
+
+type testHttpClient struct {
+	resources map[string]string
+	err       error
+}
+
+func (t *testHttpClient) GetRetry(url string) ([]byte, error) {
+	if t.err != nil {
+		return nil, t.err
+	}
+	if val, ok := t.resources[url]; ok {
+		return []byte(val), nil
+	} else {
+		return nil, pkg.ErrNotFound{fmt.Errorf("not found: %q", url)}
+	}
+}
+
+func (t *testHttpClient) Get(url string) ([]byte, error) {
+	return t.GetRetry(url)
+}
+
+func TestAvailabilityChanges(t *testing.T) {
+	want := true
+	if ac := (metadataService{}).AvailabilityChanges(); ac != want {
+		t.Fatalf("bad AvailabilityChanges: want %q, got %q", want, ac)
+	}
+}
+
+func TestType(t *testing.T) {
+	want := "ec2-metadata-service"
+	if kind := (metadataService{}).Type(); kind != want {
+		t.Fatalf("bad type: want %q, got %q", want, kind)
+	}
+}
+
+func TestIsAvailable(t *testing.T) {
+	for _, tt := range []struct {
+		root      string
+		resources map[string]string
+		expect    bool
+	}{
+		{
+			root: "/",
+			resources: map[string]string{
+				"/2009-04-04": "",
+			},
+			expect: true,
+		},
+		{
+			root:      "/",
+			resources: map[string]string{},
+			expect:    false,
+		},
+	} {
+		service := &metadataService{tt.root, &testHttpClient{tt.resources, nil}}
+		if a := service.IsAvailable(); a != tt.expect {
+			t.Fatalf("bad isAvailable (%q): want %q, got %q", tt.resources, tt.expect, a)
+		}
+	}
+}
+
+func TestFetchUserdata(t *testing.T) {
+	for _, tt := range []struct {
+		root      string
+		resources map[string]string
+		userdata  []byte
+		clientErr error
+		expectErr error
+	}{
+		{
+			root: "/",
+			resources: map[string]string{
+				"/2009-04-04/user-data": "hello",
+			},
+			userdata: []byte("hello"),
+		},
+		{
+			root:      "/",
+			clientErr: pkg.ErrNotFound{fmt.Errorf("test not found error")},
+			userdata:  []byte{},
+		},
+		{
+			root:      "/",
+			clientErr: pkg.ErrTimeout{fmt.Errorf("test timeout error")},
+			expectErr: pkg.ErrTimeout{fmt.Errorf("test timeout error")},
+		},
+	} {
+		service := &metadataService{tt.root, &testHttpClient{tt.resources, tt.clientErr}}
+		data, err := service.FetchUserdata()
+		if Error(err) != Error(tt.expectErr) {
+			t.Fatalf("bad error (%q): want %q, got %q", tt.resources, tt.expectErr, err)
+		}
+		if !bytes.Equal(data, tt.userdata) {
+			t.Fatalf("bad userdata (%q): want %q, got %q", tt.resources, tt.userdata, data)
+		}
+	}
+}
+
+func TestUrls(t *testing.T) {
+	for _, tt := range []struct {
+		root       string
+		expectRoot string
+		userdata   string
+		metadata   string
+	}{
+		{
+			root:       "/",
+			expectRoot: "/",
+			userdata:   "/2009-04-04/user-data",
+			metadata:   "/2009-04-04/meta-data",
+		},
+		{
+			root:       "http://169.254.169.254/",
+			expectRoot: "http://169.254.169.254/",
+			userdata:   "http://169.254.169.254/2009-04-04/user-data",
+			metadata:   "http://169.254.169.254/2009-04-04/meta-data",
+		},
+	} {
+		service := &metadataService{tt.root, nil}
+		if url := service.userdataUrl(); url != tt.userdata {
+			t.Fatalf("bad url (%q): want %q, got %q", tt.root, tt.userdata, url)
+		}
+		if url := service.metadataUrl(); url != tt.metadata {
+			t.Fatalf("bad url (%q): want %q, got %q", tt.root, tt.metadata, url)
+		}
+		if url := service.ConfigRoot(); url != tt.expectRoot {
+			t.Fatalf("bad url (%q): want %q, got %q", tt.root, tt.expectRoot, url)
+		}
+	}
+}
+
+func TestFetchAttributes(t *testing.T) {
+	for _, s := range []struct {
+		resources map[string]string
+		err       error
+		tests     []struct {
+			path string
+			val  []string
+		}
+	}{
+		{
+			resources: map[string]string{
+				"/":      "a\nb\nc/",
+				"/c/":    "d\ne/",
+				"/c/e/":  "f",
+				"/a":     "1",
+				"/b":     "2",
+				"/c/d":   "3",
+				"/c/e/f": "4",
+			},
+			tests: []struct {
+				path string
+				val  []string
+			}{
+				{"/", []string{"a", "b", "c/"}},
+				{"/b", []string{"2"}},
+				{"/c/d", []string{"3"}},
+				{"/c/e/", []string{"f"}},
+			},
+		},
+		{
+			err: pkg.ErrNotFound{fmt.Errorf("test error")},
+			tests: []struct {
+				path string
+				val  []string
+			}{
+				{"", nil},
+			},
+		},
+	} {
+		client := &testHttpClient{s.resources, s.err}
+		for _, tt := range s.tests {
+			attrs, err := fetchAttributes(client, tt.path)
+			if err != s.err {
+				t.Fatalf("bad error for %q (%q): want %q, got %q", tt.path, s.resources, s.err, err)
+			}
+			if !reflect.DeepEqual(attrs, tt.val) {
+				t.Fatalf("bad fetch for %q (%q): want %q, got %q", tt.path, s.resources, tt.val, attrs)
+			}
+		}
+	}
+}
+
+func TestFetchAttribute(t *testing.T) {
+	for _, s := range []struct {
+		resources map[string]string
+		err       error
+		tests     []struct {
+			path string
+			val  string
+		}
+	}{
+		{
+			resources: map[string]string{
+				"/":      "a\nb\nc/",
+				"/c/":    "d\ne/",
+				"/c/e/":  "f",
+				"/a":     "1",
+				"/b":     "2",
+				"/c/d":   "3",
+				"/c/e/f": "4",
+			},
+			tests: []struct {
+				path string
+				val  string
+			}{
+				{"/a", "1"},
+				{"/b", "2"},
+				{"/c/d", "3"},
+				{"/c/e/f", "4"},
+			},
+		},
+		{
+			err: pkg.ErrNotFound{fmt.Errorf("test error")},
+			tests: []struct {
+				path string
+				val  string
+			}{
+				{"", ""},
+			},
+		},
+	} {
+		client := &testHttpClient{s.resources, s.err}
+		for _, tt := range s.tests {
+			attr, err := fetchAttribute(client, tt.path)
+			if err != s.err {
+				t.Fatalf("bad error for %q (%q): want %q, got %q", tt.path, s.resources, s.err, err)
+			}
+			if attr != tt.val {
+				t.Fatalf("bad fetch for %q (%q): want %q, got %q", tt.path, s.resources, tt.val, attr)
+			}
+		}
+	}
+}
+
+func TestFetchMetadata(t *testing.T) {
+	for _, tt := range []struct {
+		root      string
+		resources map[string]string
+		expect    []byte
+		clientErr error
+		expectErr error
+	}{
+		{
+			root: "/",
+			resources: map[string]string{
+				"/2009-04-04/meta-data/public-keys": "bad\n",
+			},
+			expectErr: fmt.Errorf("malformed public key: \"bad\""),
+		},
+		{
+			root: "/",
+			resources: map[string]string{
+				"/2009-04-04/meta-data/hostname":                    "host",
+				"/2009-04-04/meta-data/local-ipv4":                  "1.2.3.4",
+				"/2009-04-04/meta-data/public-ipv4":                 "5.6.7.8",
+				"/2009-04-04/meta-data/public-keys":                 "0=test1\n",
+				"/2009-04-04/meta-data/public-keys/0":               "openssh-key",
+				"/2009-04-04/meta-data/public-keys/0/openssh-key":   "key",
+				"/2009-04-04/meta-data/network_config/content_path": "path",
+			},
+			expect: []byte(`{"hostname":"host","local-ipv4":"1.2.3.4","network_config":{"content_path":"path"},"public-ipv4":"5.6.7.8","public_keys":{"test1":"key"}}`),
+		},
+		{
+			clientErr: pkg.ErrTimeout{fmt.Errorf("test error")},
+			expectErr: pkg.ErrTimeout{fmt.Errorf("test error")},
+		},
+	} {
+		service := &metadataService{tt.root, &testHttpClient{tt.resources, tt.clientErr}}
+		metadata, err := service.FetchMetadata()
+		if Error(err) != Error(tt.expectErr) {
+			t.Fatalf("bad error (%q): want %q, got %q", tt.resources, tt.expectErr, err)
+		}
+		if !bytes.Equal(metadata, tt.expect) {
+			t.Fatalf("bad fetch (%q): want %q, got %q", tt.resources, tt.expect, metadata)
+		}
+	}
+}
+
+func TestNewDatasource(t *testing.T) {
+	for _, tt := range []struct {
+		root       string
+		expectRoot string
+	}{
+		{
+			root:       "",
+			expectRoot: "/",
+		},
+		{
+			root:       "/",
+			expectRoot: "/",
+		},
+		{
+			root:       "http://169.254.169.254",
+			expectRoot: "http://169.254.169.254/",
+		},
+		{
+			root:       "http://169.254.169.254/",
+			expectRoot: "http://169.254.169.254/",
+		},
+	} {
+		service := NewDatasource(tt.root)
+		if service.root != tt.expectRoot {
+			t.Fatalf("bad root (%q): want %q, got %q", tt.root, tt.expectRoot, service.root)
+		}
+	}
+}
+
+func Error(err error) string {
+	if err != nil {
+		return err.Error()
+	}
+	return ""
+}

datasource/metadata_service.go

@@ -1,153 +0,0 @@
-package datasource
-
-import (
-	"bufio"
-	"bytes"
-	"encoding/json"
-	"fmt"
-	"strings"
-
-	"github.com/coreos/coreos-cloudinit/pkg"
-)
-
-// metadataService retrieves metadata from either an OpenStack[1] (2012-08-10)
-// or EC2[2] (2009-04-04) compatible endpoint. It will first attempt to
-// directly retrieve a JSON blob from the OpenStack endpoint. If that fails
-// with a 404, it then attempts to retrieve metadata bit-by-bit from the EC2
-// endpoint, and populates that into an equivalent JSON blob. metadataService
-// also checks for userdata from EC2 and, if that fails with a 404, OpenStack.
-//
-// [1] http://docs.openstack.org/grizzly/openstack-compute/admin/content/metadata-service.html
-// [2] http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/AESDG-chapter-instancedata.html#instancedata-data-categories
-
-const (
-	BaseUrl              = "http://169.254.169.254/"
-	Ec2UserdataUrl       = BaseUrl + Ec2ApiVersion + "/user-data"
-	Ec2MetadataUrl       = BaseUrl + Ec2ApiVersion + "/meta-data"
-	OpenstackUserdataUrl = BaseUrl + "openstack/" + OpenstackApiVersion + "/user_data"
-)
-
-type metadataService struct{}
-
-type getter interface {
-	GetRetry(string) ([]byte, error)
-}
-
-func NewMetadataService() *metadataService {
-	return &metadataService{}
-}
-
-func (ms *metadataService) IsAvailable() bool {
-	client := pkg.NewHttpClient()
-	_, err := client.Get(BaseUrl)
-	return (err == nil)
-}
-
-func (ms *metadataService) AvailabilityChanges() bool {
-	return true
-}
-
-func (ms *metadataService) ConfigRoot() string {
-	return ""
-}
-
-func (ms *metadataService) FetchMetadata() ([]byte, error) {
-	return fetchMetadata(pkg.NewHttpClient())
-}
-
-func (ms *metadataService) FetchUserdata() ([]byte, error) {
-	client := pkg.NewHttpClient()
-	if data, err := client.GetRetry(Ec2UserdataUrl); err == nil {
-		return data, err
-	} else if _, ok := err.(pkg.ErrTimeout); ok {
-		return data, err
-	}
-
-	if data, err := client.GetRetry(OpenstackUserdataUrl); err == nil {
-		return data, err
-	} else if _, ok := err.(pkg.ErrNotFound); ok {
-		return []byte{}, nil
-	} else {
-		return data, err
-	}
-}
-
-func (ms *metadataService) Type() string {
-	return "metadata-service"
-}
-
-func fetchMetadata(client getter) ([]byte, error) {
-	attrs := make(map[string]interface{})
-	if keynames, err := fetchAttributes(client, fmt.Sprintf("%s/public-keys", Ec2MetadataUrl)); err == nil {
-		keyIDs := make(map[string]string)
-		for _, keyname := range keynames {
-			tokens := strings.SplitN(keyname, "=", 2)
-			if len(tokens) != 2 {
-				return nil, fmt.Errorf("malformed public key: %q\n", keyname)
-			}
-			keyIDs[tokens[1]] = tokens[0]
-		}
-
-		keys := make(map[string]string)
-		for name, id := range keyIDs {
-			sshkey, err := fetchAttribute(client, fmt.Sprintf("%s/public-keys/%s/openssh-key", Ec2MetadataUrl, id))
-			if err != nil {
-				return nil, err
-			}
-			keys[name] = sshkey
-			fmt.Printf("Found SSH key for %q\n", name)
-		}
-		attrs["public_keys"] = keys
-	} else if _, ok := err.(pkg.ErrNotFound); !ok {
-		return nil, err
-	}
-
-	if hostname, err := fetchAttribute(client, fmt.Sprintf("%s/hostname", Ec2MetadataUrl)); err == nil {
-		attrs["hostname"] = hostname
-	} else if _, ok := err.(pkg.ErrNotFound); !ok {
-		return nil, err
-	}
-
-	if localAddr, err := fetchAttribute(client, fmt.Sprintf("%s/local-ipv4", Ec2MetadataUrl)); err == nil {
-		attrs["local-ipv4"] = localAddr
-	} else if _, ok := err.(pkg.ErrNotFound); !ok {
-		return nil, err
-	}
-
-	if publicAddr, err := fetchAttribute(client, fmt.Sprintf("%s/public-ipv4", Ec2MetadataUrl)); err == nil {
-		attrs["public-ipv4"] = publicAddr
-	} else if _, ok := err.(pkg.ErrNotFound); !ok {
-		return nil, err
-	}
-
-	if content_path, err := fetchAttribute(client, fmt.Sprintf("%s/network_config/content_path", Ec2MetadataUrl)); err == nil {
-		attrs["network_config"] = map[string]string{
-			"content_path": content_path,
-		}
-	} else if _, ok := err.(pkg.ErrNotFound); !ok {
-		return nil, err
-	}
-
-	return json.Marshal(attrs)
-}
-
-func fetchAttributes(client getter, url string) ([]string, error) {
-	resp, err := client.GetRetry(url)
-	if err != nil {
-		return nil, err
-	}
-	scanner := bufio.NewScanner(bytes.NewBuffer(resp))
-	data := make([]string, 0)
-	for scanner.Scan() {
-		data = append(data, scanner.Text())
-	}
-	return data, scanner.Err()
-}
-
-func fetchAttribute(client getter, url string) (string, error) {
-	if attrs, err := fetchAttributes(client, url); err == nil && len(attrs) > 0 {
-		return attrs[0], nil
-	} else {
-		return "", err
-	}
-}

datasource/metadata_service_test.go

@@ -1,159 +0,0 @@
-package datasource
-
-import (
-	"bytes"
-	"fmt"
-	"reflect"
-	"testing"
-
-	"github.com/coreos/coreos-cloudinit/pkg"
-)
-
-type TestHttpClient struct {
-	metadata map[string]string
-	err      error
-}
-
-func (t *TestHttpClient) GetRetry(url string) ([]byte, error) {
-	if t.err != nil {
-		return nil, t.err
-	}
-	if val, ok := t.metadata[url]; ok {
-		return []byte(val), nil
-	} else {
-		return nil, pkg.ErrNotFound{fmt.Errorf("not found: %q", url)}
-	}
-}
-
-func TestMSFetchAttributes(t *testing.T) {
-	for _, s := range []struct {
-		metadata map[string]string
-		err      error
-		tests    []struct {
-			path string
-			val  []string
-		}
-	}{
-		{
-			metadata: map[string]string{
-				"/":      "a\nb\nc/",
-				"/c/":    "d\ne/",
-				"/c/e/":  "f",
-				"/a":     "1",
-				"/b":     "2",
-				"/c/d":   "3",
-				"/c/e/f": "4",
-			},
-			tests: []struct {
-				path string
-				val  []string
-			}{
-				{"/", []string{"a", "b", "c/"}},
-				{"/b", []string{"2"}},
-				{"/c/d", []string{"3"}},
-				{"/c/e/", []string{"f"}},
-			},
-		},
-		{
-			err: pkg.ErrNotFound{fmt.Errorf("test error")},
-			tests: []struct {
-				path string
-				val  []string
-			}{
-				{"", nil},
-			},
-		},
-	} {
-		client := &TestHttpClient{s.metadata, s.err}
-		for _, tt := range s.tests {
-			attrs, err := fetchAttributes(client, tt.path)
-			if err != s.err {
-				t.Fatalf("bad error for %q (%q): want %q, got %q", tt.path, s.metadata, s.err, err)
-			}
-			if !reflect.DeepEqual(attrs, tt.val) {
-				t.Fatalf("bad fetch for %q (%q): want %q, got %q", tt.path, s.metadata, tt.val, attrs)
-			}
-		}
-	}
-}
-
-func TestMSFetchAttribute(t *testing.T) {
-	for _, s := range []struct {
-		metadata map[string]string
-		err      error
-		tests    []struct {
-			path string
-			val  string
-		}
-	}{
-		{
-			metadata: map[string]string{
-				"/":      "a\nb\nc/",
-				"/c/":    "d\ne/",
-				"/c/e/":  "f",
-				"/a":     "1",
-				"/b":     "2",
-				"/c/d":   "3",
-				"/c/e/f": "4",
-			},
-			tests: []struct {
-				path string
-				val  string
-			}{
-				{"/a", "1"},
-				{"/b", "2"},
-				{"/c/d", "3"},
-				{"/c/e/f", "4"},
-			},
-		},
-		{
-			err: pkg.ErrNotFound{fmt.Errorf("test error")},
-			tests: []struct {
-				path string
-				val  string
-			}{
-				{"", ""},
-			},
-		},
-	} {
-		client := &TestHttpClient{s.metadata, s.err}
-		for _, tt := range s.tests {
-			attr, err := fetchAttribute(client, tt.path)
-			if err != s.err {
-				t.Fatalf("bad error for %q (%q): want %q, got %q", tt.path, s.metadata, s.err, err)
-			}
-			if attr != tt.val {
-				t.Fatalf("bad fetch for %q (%q): want %q, got %q", tt.path, s.metadata, tt.val, attr)
-			}
-		}
-	}
-}
-
-func TestMSFetchMetadata(t *testing.T) {
-	for _, tt := range []struct {
-		metadata map[string]string
-		err      error
-		expect   []byte
-	}{
-		{
-			metadata: map[string]string{
-				"http://169.254.169.254/2009-04-04/meta-data/hostname":                    "host",
-				"http://169.254.169.254/2009-04-04/meta-data/public-keys":                 "0=test1\n",
-				"http://169.254.169.254/2009-04-04/meta-data/public-keys/0":               "openssh-key",
-				"http://169.254.169.254/2009-04-04/meta-data/public-keys/0/openssh-key":   "key",
-				"http://169.254.169.254/2009-04-04/meta-data/network_config/content_path": "path",
-			},
-			expect: []byte(`{"hostname":"host","network_config":{"content_path":"path"},"public_keys":{"test1":"key"}}`),
-		},
-		{err: pkg.ErrTimeout{fmt.Errorf("test error")}},
-	} {
-		client := &TestHttpClient{tt.metadata, tt.err}
-		metadata, err := fetchMetadata(client)
-		if err != tt.err {
-			t.Fatalf("bad error (%q): want %q, got %q", tt.metadata, tt.err, err)
-		}
-		if !bytes.Equal(metadata, tt.expect) {
-			t.Fatalf("bad fetch (%q): want %q, got %q", tt.metadata, tt.expect, metadata)
-		}
-	}
-}

datasource/proc_cmdline/proc_cmdline.go

@@ -1,4 +1,4 @@
-package datasource
+package proc_cmdline
 
 import (
 	"errors"
@@ -18,7 +18,7 @@ type procCmdline struct {
 	Location string
 }
 
-func NewProcCmdline() *procCmdline {
+func NewDatasource() *procCmdline {
 	return &procCmdline{Location: ProcCmdlineLocation}
 }
 

datasource/proc_cmdline/proc_cmdline_test.go

@@ -1,4 +1,4 @@
-package datasource
+package proc_cmdline
 
 import (
 	"fmt"
@@ -75,7 +75,7 @@ func TestProcCmdlineAndFetchConfig(t *testing.T) {
 		t.Errorf("Test produced error: %v", err)
 	}
 
-	p := NewProcCmdline()
+	p := NewDatasource()
 	p.Location = file.Name()
 	cfg, err := p.FetchUserdata()
 	if err != nil {

datasource/url/url.go

@@ -1,4 +1,4 @@
-package datasource
+package url
 
 import "github.com/coreos/coreos-cloudinit/pkg"
 
@@ -6,7 +6,7 @@ type remoteFile struct {
 	url string
 }
 
-func NewRemoteFile(url string) *remoteFile {
+func NewDatasource(url string) *remoteFile {
 	return &remoteFile{url}
 }
 

pkg/http_client.go

@@ -57,6 +57,11 @@ type HttpClient struct {
 	client *http.Client
 }
 
+type Getter interface {
+	Get(string) ([]byte, error)
+	GetRetry(string) ([]byte, error)
+}
+
 func NewHttpClient() *HttpClient {
 	hc := &HttpClient{
 		MaxBackoff: time.Second * 5,

test

@@ -13,7 +13,16 @@ COVER=${COVER:-"-cover"}
 
 source ./build
 
-declare -a TESTPKGS=(initialize system datasource pkg network)
+declare -a TESTPKGS=(initialize
+                     system
+                     datasource
+                     datasource/configdrive
+                     datasource/file
+                     datasource/metadata/ec2
+                     datasource/proc_cmdline
+                     datasource/url
+                     pkg
+                     network)
 
 if [ -z "$PKG" ]; then
 	GOFMTPATH="$TESTPKGS coreos-cloudinit.go"