coreos-cloudinit: bump to 0.9.1

configdrive: Use the EC2 metadata over OpenStack

Documentation/cloud-config.md

@@ -1,6 +1,6 @@
 # Using Cloud-Config
 
-CoreOS allows you to declaratively customize various OS-level items, such as network configuration, user accounts, and systemd units. This document describes the full list of items we can configure. The `coreos-cloudinit` program uses these files as it configures the OS after startup or during runtime. 
+CoreOS allows you to declaratively customize various OS-level items, such as network configuration, user accounts, and systemd units. This document describes the full list of items we can configure. The `coreos-cloudinit` program uses these files as it configures the OS after startup or during runtime. Your cloud-config is processed during each boot.
 
 ## Configuration File
 
@@ -40,7 +40,7 @@ CoreOS tries to conform to each platform's native method to provide user data. E
 #### etcd
 
 The `coreos.etcd.*` parameters will be translated to a partial systemd unit acting as an etcd configuration file.
-We can use the templating feature of coreos-cloudinit to automate etcd configuration with the `$private_ipv4` and `$public_ipv4` fields. For example, the following cloud-config document...
+If the platform environment supports the templating feature of coreos-cloudinit it is possible to automate etcd configuration with the `$private_ipv4` and `$public_ipv4` fields. For example, the following cloud-config document...
 
 ```yaml
 #cloud-config
@@ -68,6 +68,8 @@ Environment="ETCD_PEER_ADDR=192.0.2.13:7001"
 For more information about the available configuration parameters, see the [etcd documentation][etcd-config].
 Note that hyphens in the coreos.etcd.* keys are mapped to underscores.
 
+_Note: The `$private_ipv4` and `$public_ipv4` substitution variables referenced in other documents are only supported on Amazon EC2, Google Compute Engine, OpenStack, Rackspace, and Vagrant._
+
 [etcd-config]: https://github.com/coreos/etcd/blob/master/Documentation/configuration.md
 
 #### fleet
@@ -123,7 +125,9 @@ coreos:
 
 #### units
 
-The `coreos.units.*` parameters define a list of arbitrary systemd units to start. Each item is an object with the following fields:
+The `coreos.units.*` parameters define a list of arbitrary systemd units to start after booting. This feature is intended to help you start essential services required to mount storage and configure networking in order to join the CoreOS cluster. It is not intended to be a Chef/Puppet replacement.
+
+Each item is an object with the following fields:
 
 - **name**: String representing unit's name. Required.
 - **runtime**: Boolean indicating whether or not to persist the unit across reboots. This is analogous to the `--runtime` argument to `systemctl enable`. Default value is false.
@@ -203,7 +207,7 @@ All but the `passwd` and `ssh-authorized-keys` fields will be ignored if the use
 - **name**: Required. Login name of user
 - **gecos**: GECOS comment of user
 - **passwd**: Hash of the password to use for this user
-- **homedir**: User's home directory. Defaults to /home/<name>
+- **homedir**: User's home directory. Defaults to /home/\<name\>
 - **no-create-home**: Boolean. Skip home directory creation.
 - **primary-group**: Default group for the user. Defaults to a new group created named after the user.
 - **groups**: Add user to these additional groups

coreos-cloudinit.go

@@ -14,7 +14,7 @@ import (
 )
 
 const (
-	version               = "0.9.0"
+	version               = "0.9.1"
 	datasourceInterval    = 100 * time.Millisecond
 	datasourceMaxInterval = 30 * time.Second
 	datasourceTimeout     = 5 * time.Minute

datasource/configdrive.go

@@ -8,10 +8,11 @@ import (
 
 type configDrive struct {
 	root     string
+	readFile func(filename string) ([]byte, error)
 }
 
 func NewConfigDrive(root string) *configDrive {
-	return &configDrive{path.Join(root, "openstack")}
+	return &configDrive{root, ioutil.ReadFile}
 }
 
 func (cd *configDrive) IsAvailable() bool {
@@ -24,23 +25,38 @@ func (cd *configDrive) AvailabilityChanges() bool {
 }
 
 func (cd *configDrive) ConfigRoot() string {
-	return cd.root
+	return cd.openstackRoot()
 }
 
+// FetchMetadata attempts to retrieve metadata from ec2/2009-04-04/meta_data.json.
 func (cd *configDrive) FetchMetadata() ([]byte, error) {
-	return cd.readFile("meta_data.json")
+	return cd.tryReadFile(path.Join(cd.ec2Root(), "meta_data.json"))
 }
 
+// FetchUserdata attempts to retrieve the userdata from ec2/2009-04-04/user_data.
+// If no data is found, it will attempt to read from openstack/latest/user_data.
 func (cd *configDrive) FetchUserdata() ([]byte, error) {
-	return cd.readFile("user_data")
+	bytes, err := cd.tryReadFile(path.Join(cd.ec2Root(), "user_data"))
+	if bytes == nil && err == nil {
+		bytes, err = cd.tryReadFile(path.Join(cd.openstackRoot(), "user_data"))
+	}
+	return bytes, err
 }
 
 func (cd *configDrive) Type() string {
 	return "cloud-drive"
 }
 
-func (cd *configDrive) readFile(filename string) ([]byte, error) {
-	data, err := ioutil.ReadFile(path.Join(cd.root, "latest", filename))
+func (cd *configDrive) ec2Root() string {
+	return path.Join(cd.root, "ec2", Ec2ApiVersion)
+}
+
+func (cd *configDrive) openstackRoot() string {
+	return path.Join(cd.root, "openstack", "latest")
+}
+
+func (cd *configDrive) tryReadFile(filename string) ([]byte, error) {
+	data, err := cd.readFile(filename)
 	if os.IsNotExist(err) {
 		err = nil
 	}

datasource/configdrive_test.go

@@ -0,0 +1,114 @@
+package datasource
+
+import (
+	"os"
+	"testing"
+)
+
+type mockFilesystem []string
+
+func (m mockFilesystem) readFile(filename string) ([]byte, error) {
+	for _, file := range m {
+		if file == filename {
+			return []byte(filename), nil
+		}
+	}
+	return nil, os.ErrNotExist
+}
+
+func TestCDFetchMetadata(t *testing.T) {
+	for _, tt := range []struct {
+		root     string
+		filename string
+		files    mockFilesystem
+	}{
+		{
+			"/",
+			"",
+			mockFilesystem{},
+		},
+		{
+			"/",
+			"/ec2/2009-04-04/meta_data.json",
+			mockFilesystem([]string{"/ec2/2009-04-04/meta_data.json"}),
+		},
+		{
+			"/media/configdrive",
+			"/media/configdrive/ec2/2009-04-04/meta_data.json",
+			mockFilesystem([]string{"/media/configdrive/ec2/2009-04-04/meta_data.json"}),
+		},
+	} {
+		cd := configDrive{tt.root, tt.files.readFile}
+		filename, err := cd.FetchMetadata()
+		if err != nil {
+			t.Fatalf("bad error for %q: want %q, got %q", tt, nil, err)
+		}
+		if string(filename) != tt.filename {
+			t.Fatalf("bad path for %q: want %q, got %q", tt, tt.filename, filename)
+		}
+	}
+}
+
+func TestCDFetchUserdata(t *testing.T) {
+	for _, tt := range []struct {
+		root     string
+		filename string
+		files    mockFilesystem
+	}{
+		{
+			"/",
+			"",
+			mockFilesystem{},
+		},
+		{
+			"/",
+			"/ec2/2009-04-04/user_data",
+			mockFilesystem([]string{"/ec2/2009-04-04/user_data"}),
+		},
+		{
+			"/",
+			"/openstack/latest/user_data",
+			mockFilesystem([]string{"/openstack/latest/user_data"}),
+		},
+		{
+			"/",
+			"/ec2/2009-04-04/user_data",
+			mockFilesystem([]string{"/openstack/latest/user_data", "/ec2/2009-04-04/user_data"}),
+		},
+		{
+			"/media/configdrive",
+			"/media/configdrive/ec2/2009-04-04/user_data",
+			mockFilesystem([]string{"/media/configdrive/ec2/2009-04-04/user_data"}),
+		},
+	} {
+		cd := configDrive{tt.root, tt.files.readFile}
+		filename, err := cd.FetchUserdata()
+		if err != nil {
+			t.Fatalf("bad error for %q: want %q, got %q", tt, nil, err)
+		}
+		if string(filename) != tt.filename {
+			t.Fatalf("bad path for %q: want %q, got %q", tt, tt.filename, filename)
+		}
+	}
+}
+
+func TestCDConfigRoot(t *testing.T) {
+	for _, tt := range []struct {
+		root       string
+		configRoot string
+	}{
+		{
+			"/",
+			"/openstack/latest",
+		},
+		{
+			"/media/configdrive",
+			"/media/configdrive/openstack/latest",
+		},
+	} {
+		cd := configDrive{tt.root, nil}
+		if configRoot := cd.ConfigRoot(); configRoot != tt.configRoot {
+			t.Fatalf("bad config root for %q: want %q, got %q", tt, tt.configRoot, configRoot)
+		}
+	}
+}

datasource/datasource.go

@@ -1,5 +1,10 @@
 package datasource
 
+const (
+	Ec2ApiVersion       = "2009-04-04"
+	OpenstackApiVersion = "2012-08-10"
+)
+
 type Datasource interface {
 	IsAvailable() bool
 	AvailabilityChanges() bool

datasource/metadata_service.go

@@ -22,11 +22,9 @@ import (
 
 const (
 	BaseUrl              = "http://169.254.169.254/"
-	Ec2ApiVersion        = "2009-04-04"
 	Ec2UserdataUrl       = BaseUrl + Ec2ApiVersion + "/user-data"
 	Ec2MetadataUrl       = BaseUrl + Ec2ApiVersion + "/meta-data"
-	OpenstackApiVersion  = "openstack/2012-08-10"
-	OpenstackUserdataUrl = BaseUrl + OpenstackApiVersion + "/user_data"
+	OpenstackUserdataUrl = BaseUrl + "openstack/" + OpenstackApiVersion + "/user_data"
 )
 
 type metadataService struct{}

datasource/metadata_service_test.go

@@ -25,7 +25,7 @@ func (t *TestHttpClient) GetRetry(url string) ([]byte, error) {
 	}
 }
 
-func TestFetchAttributes(t *testing.T) {
+func TestMSFetchAttributes(t *testing.T) {
 	for _, s := range []struct {
 		metadata map[string]string
 		err      error
@@ -77,7 +77,7 @@ func TestFetchAttributes(t *testing.T) {
 	}
 }
 
-func TestFetchAttribute(t *testing.T) {
+func TestMSFetchAttribute(t *testing.T) {
 	for _, s := range []struct {
 		metadata map[string]string
 		err      error
@@ -129,7 +129,7 @@ func TestFetchAttribute(t *testing.T) {
 	}
 }
 
-func TestFetchMetadata(t *testing.T) {
+func TestMSFetchMetadata(t *testing.T) {
 	for _, tt := range []struct {
 		metadata map[string]string
 		err      error

initialize/config.go

@@ -66,52 +66,63 @@ func warnOnUnrecognizedKeys(contents string, warn warner) {
 	}
 
 	// Check for unrecognized coreos options, if any are set
-	coreos, ok := c["coreos"]
-	if ok {
-		set := coreos.(map[interface{}]interface{})
+	if coreos, ok := c["coreos"]; ok {
+		if set, ok := coreos.(map[interface{}]interface{}); ok {
 			known := cc["coreos"].(map[interface{}]interface{})
 			for k, _ := range set {
-			key := k.(string)
+				if key, ok := k.(string); ok {
 					if _, ok := known[key]; !ok {
 						warn("Warning: unrecognized key %q in coreos section of provided cloud config - ignoring", key)
 					}
+				} else {
+					warn("Warning: unrecognized key %q in coreos section of provided cloud config - ignoring", k)
+				}
+			}
 		}
 	}
 
 	// Check for any badly-specified users, if any are set
-	users, ok := c["users"]
-	if ok {
+	if users, ok := c["users"]; ok {
 		var known map[string]interface{}
 		b, _ := goyaml.Marshal(&system.User{})
 		goyaml.Unmarshal(b, &known)
 
-		set := users.([]interface{})
+		if set, ok := users.([]interface{}); ok {
 			for _, u := range set {
-			user := u.(map[interface{}]interface{})
+				if user, ok := u.(map[interface{}]interface{}); ok {
 					for k, _ := range user {
-				key := k.(string)
+						if key, ok := k.(string); ok {
 							if _, ok := known[key]; !ok {
 								warn("Warning: unrecognized key %q in user section of cloud config - ignoring", key)
 							}
+						} else {
+							warn("Warning: unrecognized key %q in user section of cloud config - ignoring", k)
+						}
+					}
+				}
 			}
 		}
 	}
 
 	// Check for any badly-specified files, if any are set
-	files, ok := c["write_files"]
-	if ok {
+	if files, ok := c["write_files"]; ok {
 		var known map[string]interface{}
 		b, _ := goyaml.Marshal(&system.File{})
 		goyaml.Unmarshal(b, &known)
 
-		set := files.([]interface{})
+		if set, ok := files.([]interface{}); ok {
 			for _, f := range set {
-			file := f.(map[interface{}]interface{})
+				if file, ok := f.(map[interface{}]interface{}); ok {
 					for k, _ := range file {
-				key := k.(string)
+						if key, ok := k.(string); ok {
 							if _, ok := known[key]; !ok {
 								warn("Warning: unrecognized key %q in file section of cloud config - ignoring", key)
 							}
+						} else {
+							warn("Warning: unrecognized key %q in file section of cloud config - ignoring", k)
+						}
+					}
+				}
 			}
 		}
 	}

initialize/config_test.go

@@ -8,6 +8,34 @@ import (
 	"github.com/coreos/coreos-cloudinit/system"
 )
 
+func TestCloudConfigInvalidKeys(t *testing.T) {
+	defer func() {
+		if r := recover(); r != nil {
+			t.Fatalf("panic while instantiating CloudConfig with nil keys: %v", r)
+		}
+	}()
+
+	for _, tt := range []struct {
+		contents string
+	}{
+		{"coreos:"},
+		{"ssh_authorized_keys:"},
+		{"ssh_authorized_keys:\n  -"},
+		{"ssh_authorized_keys:\n  - 0:"},
+		{"write_files:"},
+		{"write_files:\n  -"},
+		{"write_files:\n  - 0:"},
+		{"users:"},
+		{"users:\n  -"},
+		{"users:\n  - 0:"},
+	} {
+		_, err := NewCloudConfig(tt.contents)
+		if err != nil {
+			t.Fatalf("error instantiating CloudConfig with invalid keys: %v", err)
+		}
+	}
+}
+
 func TestCloudConfigUnknownKeys(t *testing.T) {
 	contents := `
 coreos: