coreos-cloudinit: bump to 0.8.9

env_file: fix broken test cases

Documentation/cloud-config.md

@@ -16,7 +16,7 @@ We've designed our implementation to allow the same cloud-config file to work ac
 
 The cloud-config file uses the [YAML][yaml] file format, which uses whitespace and new-lines to delimit lists, associative arrays, and values.
 
-A cloud-config file should contain an associative array which has zero or more of the following keys:
+A cloud-config file should contain `#cloud-config`, followed by an associative array which has zero or more of the following keys:
 
 - `coreos`
 - `ssh_authorized_keys`

coreos-cloudinit.go

@@ -14,7 +14,7 @@ import (
 )
 
 const (
-	version               = "0.8.0"
+	version               = "0.8.9"
 	datasourceInterval    = 100 * time.Millisecond
 	datasourceMaxInterval = 30 * time.Second
 	datasourceTimeout     = 5 * time.Minute
@@ -102,6 +102,7 @@ func main() {
 		die()
 	}
 
+	// Extract IPv4 addresses from metadata if possible
 	var subs map[string]string
 	if len(metadataBytes) > 0 {
 		subs, err = initialize.ExtractIPsFromMetadata(metadataBytes)
@@ -110,29 +111,88 @@ func main() {
 			die()
 		}
 	}
+
+	// Apply environment to user-data
 	env := initialize.NewEnvironment("/", ds.ConfigRoot(), workspace, convertNetconf, sshKeyName, subs)
+	userdata := env.Apply(string(userdataBytes))
 
-	if len(metadataBytes) > 0 {
-		if err := processMetadata(string(metadataBytes), env); err != nil {
-			fmt.Printf("Failed to process meta-data: %v\n", err)
+	var ccm, ccu *initialize.CloudConfig
+	var script *system.Script
+	if ccm, err = initialize.ParseMetaData(string(metadataBytes)); err != nil {
+		fmt.Printf("Failed to parse meta-data: %v\n", err)
 		die()
 	}
+	if ud, err := initialize.ParseUserData(userdata); err != nil {
+		fmt.Printf("Failed to parse user-data: %v\n", err)
+		die()
 	} else {
-		fmt.Println("No meta-data to handle.")
+		switch t := ud.(type) {
+		case *initialize.CloudConfig:
+			ccu = t
+		case system.Script:
+			script = &t
+		}
 	}
 
-	if len(userdataBytes) > 0 {
-		if err := processUserdata(string(userdataBytes), env); err != nil {
-			fmt.Printf("Failed to process user-data: %v\n", err)
-			if !ignoreFailure {
+	var cc *initialize.CloudConfig
+	if ccm != nil && ccu != nil {
+		fmt.Println("Merging cloud-config from meta-data and user-data")
+		merged := mergeCloudConfig(*ccm, *ccu)
+		cc = &merged
+	} else if ccm != nil && ccu == nil {
+		fmt.Println("Processing cloud-config from meta-data")
+		cc = ccm
+	} else if ccm == nil && ccu != nil {
+		fmt.Println("Processing cloud-config from user-data")
+		cc = ccu
+	} else {
+		fmt.Println("No cloud-config data to handle.")
+	}
+
+	if cc != nil {
+		if err = initialize.Apply(*cc, env); err != nil {
+			fmt.Printf("Failed to apply cloud-config: %v\n", err)
 			die()
 		}
 	}
-	} else {
-		fmt.Println("No user-data to handle.")
+
+	if script != nil {
+		if err = runScript(*script, env); err != nil {
+			fmt.Printf("Failed to run script: %v\n", err)
+			die()
+		}
 	}
 }
 
+// mergeCloudConfig merges certain options from mdcc (a CloudConfig derived from
+// meta-data) onto udcc (a CloudConfig derived from user-data), if they are
+// not already set on udcc (i.e. user-data always takes precedence)
+// NB: This needs to be kept in sync with ParseMetadata so that it tracks all
+// elements of a CloudConfig which that function can populate.
+func mergeCloudConfig(mdcc, udcc initialize.CloudConfig) (cc initialize.CloudConfig) {
+	if mdcc.Hostname != "" {
+		if udcc.Hostname != "" {
+			fmt.Printf("Warning: user-data hostname (%s) overrides metadata hostname (%s)", udcc.Hostname, mdcc.Hostname)
+		} else {
+			udcc.Hostname = mdcc.Hostname
+		}
+
+	}
+	for _, key := range mdcc.SSHAuthorizedKeys {
+		udcc.SSHAuthorizedKeys = append(udcc.SSHAuthorizedKeys, key)
+	}
+	if mdcc.NetworkConfigPath != "" {
+		if udcc.NetworkConfigPath != "" {
+			fmt.Printf("Warning: user-data NetworkConfigPath %s overrides metadata NetworkConfigPath %s", udcc.NetworkConfigPath, mdcc.NetworkConfigPath)
+		} else {
+			udcc.NetworkConfigPath = mdcc.NetworkConfigPath
+		}
+	}
+	return udcc
+}
+
+// getDatasources creates a slice of possible Datasources for cloudinit based
+// on the different source command-line flags.
 func getDatasources() []datasource.Datasource {
 	dss := make([]datasource.Datasource, 0, 5)
 	if sources.file != "" {
@@ -153,6 +213,11 @@ func getDatasources() []datasource.Datasource {
 	return dss
 }
 
+// selectDatasource attempts to choose a valid Datasource to use based on its
+// current availability. The first Datasource to report to be available is
+// returned. Datasources will be retried if possible if they are not
+// immediately available. If all Datasources are permanently unavailable or
+// datasourceTimeout is reached before one becomes available, nil is returned.
 func selectDatasource(sources []datasource.Datasource) datasource.Datasource {
 	ds := make(chan datasource.Datasource)
 	stop := make(chan struct{})
@@ -199,48 +264,18 @@ func selectDatasource(sources []datasource.Datasource) datasource.Datasource {
 	return s
 }
 
-func processUserdata(userdata string, env *initialize.Environment) error {
-	userdata = env.Apply(userdata)
-
-	parsed, err := initialize.ParseUserData(userdata)
-	if err != nil {
-		fmt.Printf("Failed parsing user-data: %v\n", err)
-		return err
-	}
-
-	err = initialize.PrepWorkspace(env.Workspace())
+// TODO(jonboulle): this should probably be refactored and moved into a different module
+func runScript(script system.Script, env *initialize.Environment) error {
+	err := initialize.PrepWorkspace(env.Workspace())
 	if err != nil {
 		fmt.Printf("Failed preparing workspace: %v\n", err)
 		return err
 	}
-
-	switch t := parsed.(type) {
-	case initialize.CloudConfig:
-		err = initialize.Apply(t, env)
-	case system.Script:
-		var path string
-		path, err = initialize.PersistScriptInWorkspace(t, env.Workspace())
+	path, err := initialize.PersistScriptInWorkspace(script, env.Workspace())
 	if err == nil {
 		var name string
 		name, err = system.ExecuteScript(path)
 		initialize.PersistUnitNameInWorkspace(name, env.Workspace())
 	}
-	}
-
 	return err
 }
-
-func processMetadata(metadata string, env *initialize.Environment) error {
-	parsed, err := initialize.ParseMetaData(metadata)
-	if err != nil {
-		fmt.Printf("Failed parsing meta-data: %v\n", err)
-		return err
-	}
-	err = initialize.PrepWorkspace(env.Workspace())
-	if err != nil {
-		fmt.Printf("Failed preparing workspace: %v\n", err)
-		return err
-	}
-
-	return initialize.Apply(parsed, env)
-}

coreos-cloudinit_test.go

@@ -0,0 +1,110 @@
+package main
+
+import (
+	"reflect"
+	"testing"
+
+	"github.com/coreos/coreos-cloudinit/initialize"
+)
+
+func TestMergeCloudConfig(t *testing.T) {
+	simplecc := initialize.CloudConfig{
+		SSHAuthorizedKeys: []string{"abc", "def"},
+		Hostname:          "foobar",
+		NetworkConfigPath: "/path/somewhere",
+	}
+	for i, tt := range []struct {
+		udcc initialize.CloudConfig
+		mdcc initialize.CloudConfig
+		want initialize.CloudConfig
+	}{
+		{
+			// If mdcc is empty, udcc should be returned unchanged
+			simplecc,
+			initialize.CloudConfig{},
+			simplecc,
+		},
+		{
+			// If udcc is empty, mdcc should be returned unchanged(overridden)
+			initialize.CloudConfig{},
+			simplecc,
+			simplecc,
+		},
+		{
+			// user-data should override completely in the case of conflicts
+			simplecc,
+			initialize.CloudConfig{
+				Hostname:          "meta-hostname",
+				NetworkConfigPath: "/path/meta",
+			},
+			simplecc,
+		},
+		{
+			// Mixed merge should succeed
+			initialize.CloudConfig{
+				SSHAuthorizedKeys: []string{"abc", "def"},
+				Hostname:          "user-hostname",
+				NetworkConfigPath: "/path/somewhere",
+			},
+			initialize.CloudConfig{
+				SSHAuthorizedKeys: []string{"woof", "qux"},
+				Hostname:          "meta-hostname",
+			},
+			initialize.CloudConfig{
+				SSHAuthorizedKeys: []string{"abc", "def", "woof", "qux"},
+				Hostname:          "user-hostname",
+				NetworkConfigPath: "/path/somewhere",
+			},
+		},
+		{
+			// Completely non-conflicting merge should be fine
+			initialize.CloudConfig{
+				Hostname: "supercool",
+			},
+			initialize.CloudConfig{
+				SSHAuthorizedKeys: []string{"zaphod", "beeblebrox"},
+				NetworkConfigPath: "/dev/fun",
+			},
+			initialize.CloudConfig{
+				Hostname:          "supercool",
+				SSHAuthorizedKeys: []string{"zaphod", "beeblebrox"},
+				NetworkConfigPath: "/dev/fun",
+			},
+		},
+		{
+			// Non-mergeable settings in user-data should not be affected
+			initialize.CloudConfig{
+				Hostname:       "mememe",
+				ManageEtcHosts: initialize.EtcHosts("lolz"),
+			},
+			initialize.CloudConfig{
+				Hostname:          "youyouyou",
+				NetworkConfigPath: "meta-meta-yo",
+			},
+			initialize.CloudConfig{
+				Hostname:          "mememe",
+				ManageEtcHosts:    initialize.EtcHosts("lolz"),
+				NetworkConfigPath: "meta-meta-yo",
+			},
+		},
+		{
+			// Non-mergeable (unexpected) settings in meta-data are ignored
+			initialize.CloudConfig{
+				Hostname: "mememe",
+			},
+			initialize.CloudConfig{
+				ManageEtcHosts:    initialize.EtcHosts("lolz"),
+				NetworkConfigPath: "meta-meta-yo",
+			},
+			initialize.CloudConfig{
+				Hostname:          "mememe",
+				NetworkConfigPath: "meta-meta-yo",
+			},
+		},
+	} {
+		got := mergeCloudConfig(tt.mdcc, tt.udcc)
+		if !reflect.DeepEqual(got, tt.want) {
+			t.Errorf("case #%d: mergeCloudConfig mutated CloudConfig unexpectedly:\ngot:\n%s\nwant:\n%s", i, got, tt.want)
+		}
+	}
+}

datasource/metadata_service.go

@@ -4,6 +4,7 @@ import (
 	"bufio"
 	"bytes"
 	"encoding/json"
+	"fmt"
 	"strings"
 
 	"github.com/coreos/coreos-cloudinit/pkg"
@@ -23,10 +24,9 @@ const (
 	BaseUrl              = "http://169.254.169.254/"
 	Ec2ApiVersion        = "2009-04-04"
 	Ec2UserdataUrl       = BaseUrl + Ec2ApiVersion + "/user-data"
-	Ec2MetadataUrl       = BaseUrl + Ec2ApiVersion + "/meta-data/"
+	Ec2MetadataUrl       = BaseUrl + Ec2ApiVersion + "/meta-data"
 	OpenstackApiVersion  = "openstack/2012-08-10"
 	OpenstackUserdataUrl = BaseUrl + OpenstackApiVersion + "/user_data"
-	OpenstackMetadataUrl = BaseUrl + OpenstackApiVersion + "/meta_data.json"
 )
 
 type metadataService struct{}
@@ -64,7 +64,14 @@ func (ms *metadataService) FetchUserdata() ([]byte, error) {
 	} else if _, ok := err.(pkg.ErrTimeout); ok {
 		return data, err
 	}
-	return client.GetRetry(OpenstackUserdataUrl)
+
+	if data, err := client.GetRetry(OpenstackUserdataUrl); err == nil {
+		return data, err
+	} else if _, ok := err.(pkg.ErrNotFound); ok {
+		return []byte{}, nil
+	} else {
+		return data, err
+	}
 }
 
 func (ms *metadataService) Type() string {
@@ -72,16 +79,57 @@ func (ms *metadataService) Type() string {
 }
 
 func fetchMetadata(client getter) ([]byte, error) {
-	if metadata, err := client.GetRetry(OpenstackMetadataUrl); err == nil {
-		return metadata, nil
-	} else if _, ok := err.(pkg.ErrTimeout); ok {
-		return nil, err
+	attrs := make(map[string]interface{})
+	if keynames, err := fetchAttributes(client, fmt.Sprintf("%s/public-keys", Ec2MetadataUrl)); err == nil {
+		keyIDs := make(map[string]string)
+		for _, keyname := range keynames {
+			tokens := strings.SplitN(keyname, "=", 2)
+			if len(tokens) != 2 {
+				return nil, fmt.Errorf("malformed public key: %q\n", keyname)
+			}
+			keyIDs[tokens[1]] = tokens[0]
 		}
 
-	attrs, err := fetchChildAttributes(client, Ec2MetadataUrl)
+		keys := make(map[string]string)
+		for name, id := range keyIDs {
+			sshkey, err := fetchAttribute(client, fmt.Sprintf("%s/public-keys/%s/openssh-key", Ec2MetadataUrl, id))
 			if err != nil {
 				return nil, err
 			}
+			keys[name] = sshkey
+			fmt.Printf("Found SSH key for %q\n", name)
+		}
+		attrs["public_keys"] = keys
+	} else if _, ok := err.(pkg.ErrNotFound); !ok {
+		return nil, err
+	}
+
+	if hostname, err := fetchAttribute(client, fmt.Sprintf("%s/hostname", Ec2MetadataUrl)); err == nil {
+		attrs["hostname"] = hostname
+	} else if _, ok := err.(pkg.ErrNotFound); !ok {
+		return nil, err
+	}
+
+	if localAddr, err := fetchAttribute(client, fmt.Sprintf("%s/local-ipv4", Ec2MetadataUrl)); err == nil {
+		attrs["local-ipv4"] = localAddr
+	} else if _, ok := err.(pkg.ErrNotFound); !ok {
+		return nil, err
+	}
+
+	if publicAddr, err := fetchAttribute(client, fmt.Sprintf("%s/public-ipv4", Ec2MetadataUrl)); err == nil {
+		attrs["public-ipv4"] = publicAddr
+	} else if _, ok := err.(pkg.ErrNotFound); !ok {
+		return nil, err
+	}
+
+	if content_path, err := fetchAttribute(client, fmt.Sprintf("%s/network_config/content_path", Ec2MetadataUrl)); err == nil {
+		attrs["network_config"] = map[string]string{
+			"content_path": content_path,
+		}
+	} else if _, ok := err.(pkg.ErrNotFound); !ok {
+		return nil, err
+	}
+
 	return json.Marshal(attrs)
 }
 
@@ -93,37 +141,15 @@ func fetchAttributes(client getter, url string) ([]string, error) {
 	scanner := bufio.NewScanner(bytes.NewBuffer(resp))
 	data := make([]string, 0)
 	for scanner.Scan() {
-		data = append(data, strings.Split(scanner.Text(), "=")[0])
+		data = append(data, scanner.Text())
 	}
 	return data, scanner.Err()
 }
 
-func fetchAttribute(client getter, url string) (interface{}, error) {
+func fetchAttribute(client getter, url string) (string, error) {
 	if attrs, err := fetchAttributes(client, url); err == nil && len(attrs) > 0 {
 		return attrs[0], nil
 	} else {
 		return "", err
 	}
 }
-
-func fetchChildAttributes(client getter, url string) (interface{}, error) {
-	attrs := make(map[string]interface{})
-	attrList, err := fetchAttributes(client, url)
-	if err != nil {
-		return nil, err
-	}
-	for _, attr := range attrList {
-		var fetchFunc func(getter, string) (interface{}, error)
-		if strings.HasSuffix(attr, "/") {
-			fetchFunc = fetchChildAttributes
-		} else {
-			fetchFunc = fetchAttribute
-		}
-		if value, err := fetchFunc(client, url+attr); err == nil {
-			attrs[strings.TrimSuffix(attr, "/")] = value
-		} else {
-			return nil, err
-		}
-	}
-	return attrs, nil
-}

datasource/metadata_service_test.go

@@ -137,24 +137,15 @@ func TestFetchMetadata(t *testing.T) {
 	}{
 		{
 			metadata: map[string]string{
-				"http://169.254.169.254/2009-04-04/meta-data/":      "a\nb\nc/",
-				"http://169.254.169.254/2009-04-04/meta-data/c/":    "d\ne/",
-				"http://169.254.169.254/2009-04-04/meta-data/c/e/":  "f",
-				"http://169.254.169.254/2009-04-04/meta-data/a":     "1",
-				"http://169.254.169.254/2009-04-04/meta-data/b":     "2",
-				"http://169.254.169.254/2009-04-04/meta-data/c/d":   "3",
-				"http://169.254.169.254/2009-04-04/meta-data/c/e/f": "4",
+				"http://169.254.169.254/2009-04-04/meta-data/hostname":                    "host",
+				"http://169.254.169.254/2009-04-04/meta-data/public-keys":                 "0=test1\n",
+				"http://169.254.169.254/2009-04-04/meta-data/public-keys/0":               "openssh-key",
+				"http://169.254.169.254/2009-04-04/meta-data/public-keys/0/openssh-key":   "key",
+				"http://169.254.169.254/2009-04-04/meta-data/network_config/content_path": "path",
 			},
-			expect: []byte(`{"a":"1","b":"2","c":{"d":"3","e":{"f":"4"}}}`),
-		},
-		{
-			metadata: map[string]string{
-				"http://169.254.169.254/openstack/2012-08-10/meta_data.json": "test",
-			},
-			expect: []byte("test"),
+			expect: []byte(`{"hostname":"host","network_config":{"content_path":"path"},"public_keys":{"test1":"key"}}`),
 		},
 		{err: pkg.ErrTimeout{fmt.Errorf("test error")}},
-		{err: pkg.ErrNotFound{fmt.Errorf("test error")}},
 	} {
 		client := &TestHttpClient{tt.metadata, tt.err}
 		metadata, err := fetchMetadata(client)

initialize/config.go

@@ -223,12 +223,27 @@ func Apply(cfg CloudConfig, env *Environment) error {
 		cfg.Coreos.Units = append(cfg.Coreos.Units, u...)
 	}
 
+	wroteEnvironment := false
 	for _, file := range cfg.WriteFiles {
-		path, err := system.WriteFile(&file, env.Root())
+		fullPath, err := system.WriteFile(&file, env.Root())
 		if err != nil {
 			return err
 		}
-		log.Printf("Wrote file %s to filesystem", path)
+		if path.Clean(file.Path) == "/etc/environment" {
+			wroteEnvironment = true
+		}
+		log.Printf("Wrote file %s to filesystem", fullPath)
+	}
+
+	if !wroteEnvironment {
+		ef := env.DefaultEnvironmentFile()
+		if ef != nil {
+			err := system.WriteEnvFile(ef, env.Root())
+			if err != nil {
+				return err
+			}
+			log.Printf("Updated /etc/environment")
+		}
 	}
 
 	if env.NetconfType() != "" {

initialize/env.go

@@ -4,6 +4,8 @@ import (
 	"os"
 	"path"
 	"strings"
+
+	"github.com/coreos/coreos-cloudinit/system"
 )
 
 const DefaultSSHKeyName = "coreos-cloudinit"
@@ -65,6 +67,26 @@ func (e *Environment) Apply(data string) string {
 	return data
 }
 
+func (e *Environment) DefaultEnvironmentFile() *system.EnvFile {
+	ef := system.EnvFile{
+		File: &system.File{
+			Path: "/etc/environment",
+		},
+		Vars: map[string]string{},
+	}
+	if ip, ok := e.substitutions["$public_ipv4"]; ok && len(ip) > 0 {
+		ef.Vars["COREOS_PUBLIC_IPV4"] = ip
+	}
+	if ip, ok := e.substitutions["$private_ipv4"]; ok && len(ip) > 0 {
+		ef.Vars["COREOS_PRIVATE_IPV4"] = ip
+	}
+	if len(ef.Vars) == 0 {
+		return nil
+	} else {
+		return &ef
+	}
+}
+
 // normalizeSvcEnv standardizes the keys of the map (environment variables for a service)
 // by replacing any dashes with underscores and ensuring they are entirely upper case.
 // For example, "some-env" --> "SOME_ENV"

initialize/env_test.go

@@ -1,8 +1,12 @@
 package initialize
 
 import (
+	"io/ioutil"
 	"os"
+	"path"
 	"testing"
+
+	"github.com/coreos/coreos-cloudinit/system"
 )
 
 func TestEnvironmentApply(t *testing.T) {
@@ -56,3 +60,47 @@ ExecStop=/usr/bin/echo $unknown`,
 		}
 	}
 }
+
+func TestEnvironmentFile(t *testing.T) {
+	subs := map[string]string{
+		"$public_ipv4":  "1.2.3.4",
+		"$private_ipv4": "5.6.7.8",
+	}
+	expect := "COREOS_PUBLIC_IPV4=1.2.3.4\nCOREOS_PRIVATE_IPV4=5.6.7.8\n"
+
+	dir, err := ioutil.TempDir(os.TempDir(), "coreos-cloudinit-")
+	if err != nil {
+		t.Fatalf("Unable to create tempdir: %v", err)
+	}
+	defer os.RemoveAll(dir)
+
+	env := NewEnvironment("./", "./", "./", "", "", subs)
+	ef := env.DefaultEnvironmentFile()
+	err = system.WriteEnvFile(ef, dir)
+	if err != nil {
+		t.Fatalf("WriteEnvFile failed: %v", err)
+	}
+
+	fullPath := path.Join(dir, "etc", "environment")
+	contents, err := ioutil.ReadFile(fullPath)
+	if err != nil {
+		t.Fatalf("Unable to read expected file: %v", err)
+	}
+
+	if string(contents) != expect {
+		t.Fatalf("File has incorrect contents: %q", contents)
+	}
+}
+
+func TestEnvironmentFileNil(t *testing.T) {
+	subs := map[string]string{
+		"$public_ipv4":  "",
+		"$private_ipv4": "",
+	}
+
+	env := NewEnvironment("./", "./", "./", "", "", subs)
+	ef := env.DefaultEnvironmentFile()
+	if ef != nil {
+		t.Fatalf("Environment file not nil: %v", ef)
+	}
+}

initialize/meta_data.go

@@ -3,8 +3,11 @@ package initialize
 import "encoding/json"
 
 // ParseMetaData parses a JSON blob in the OpenStack metadata service format, and
-// converts it to a CloudConfig
-func ParseMetaData(contents string) (cfg CloudConfig, err error) {
+// converts it to a partially hydrated CloudConfig
+func ParseMetaData(contents string) (*CloudConfig, error) {
+	if len(contents) == 0 {
+		return nil, nil
+	}
 	var metadata struct {
 		SSHAuthorizedKeyMap map[string]string `json:"public_keys"`
 		Hostname            string            `json:"hostname"`
@@ -12,17 +15,20 @@ func ParseMetaData(contents string) (cfg CloudConfig, err error) {
 			ContentPath string `json:"content_path"`
 		} `json:"network_config"`
 	}
-	if err = json.Unmarshal([]byte(contents), &metadata); err != nil {
-		return
+	if err := json.Unmarshal([]byte(contents), &metadata); err != nil {
+		return nil, err
 	}
 
+	var cfg CloudConfig
+	if len(metadata.SSHAuthorizedKeyMap) > 0 {
 		cfg.SSHAuthorizedKeys = make([]string, 0, len(metadata.SSHAuthorizedKeyMap))
 		for _, key := range metadata.SSHAuthorizedKeyMap {
 			cfg.SSHAuthorizedKeys = append(cfg.SSHAuthorizedKeys, key)
 		}
+	}
 	cfg.Hostname = metadata.Hostname
 	cfg.NetworkConfigPath = metadata.NetworkConfig.ContentPath
-	return
+	return &cfg, nil
 }
 
 // ExtractIPsFromMetaData parses a JSON blob in the OpenStack metadata service format,

initialize/meta_data_test.go

@@ -3,6 +3,39 @@ package initialize
 import "reflect"
 import "testing"
 
+func TestParseMetadata(t *testing.T) {
+	for i, tt := range []struct {
+		in   string
+		want *CloudConfig
+		err  bool
+	}{
+		{"", nil, false},
+		{`garbage, invalid json`, nil, true},
+		{`{"foo": "bar"}`, &CloudConfig{}, false},
+		{`{"network_config": {"content_path": "asdf"}}`, &CloudConfig{NetworkConfigPath: "asdf"}, false},
+		{`{"hostname": "turkleton"}`, &CloudConfig{Hostname: "turkleton"}, false},
+		{`{"public_keys": {"jack": "jill", "bob": "alice"}}`, &CloudConfig{SSHAuthorizedKeys: []string{"jill", "alice"}}, false},
+		{`{"unknown": "thing", "hostname": "my_host", "public_keys": {"do": "re", "mi": "fa"}, "network_config": {"content_path": "/root", "blah": "zzz"}}`, &CloudConfig{SSHAuthorizedKeys: []string{"re", "fa"}, Hostname: "my_host", NetworkConfigPath: "/root"}, false},
+	} {
+		got, err := ParseMetaData(tt.in)
+		if tt.err != (err != nil) {
+			t.Errorf("case #%d: bad error state: got %t, want %t (err=%v)", i, (err != nil), tt.err, err)
+		}
+		if got == nil {
+			if tt.want != nil {
+				t.Errorf("case #%d: unexpected nil output", i)
+			}
+		} else if tt.want == nil {
+			t.Errorf("case #%d: unexpected non-nil output", i)
+		} else {
+			if !reflect.DeepEqual(*got, *tt.want) {
+				t.Errorf("case #%d: bad output:\ngot\n%v\nwant\n%v", i, *got, *tt.want)
+			}
+		}
+	}
+
+}
+
 func TestExtractIPsFromMetadata(t *testing.T) {
 	for i, tt := range []struct {
 		in  []byte

initialize/user_data.go

@@ -9,6 +9,9 @@ import (
 )
 
 func ParseUserData(contents string) (interface{}, error) {
+	if len(contents) == 0 {
+		return nil, nil
+	}
 	header := strings.SplitN(contents, "\n", 2)[0]
 
 	// Explicitly trim the header so we can handle user-data from
@@ -19,14 +22,9 @@ func ParseUserData(contents string) (interface{}, error) {
 	if strings.HasPrefix(header, "#!") {
 		log.Printf("Parsing user-data as script")
 		return system.Script(contents), nil
-
 	} else if header == "#cloud-config" {
 		log.Printf("Parsing user-data as cloud-config")
-		cfg, err := NewCloudConfig(contents)
-		if err != nil {
-			log.Fatal(err.Error())
-		}
-		return *cfg, nil
+		return NewCloudConfig(contents)
 	} else {
 		return nil, fmt.Errorf("Unrecognized user-data header: %s", header)
 	}

initialize/user_data_test.go

@@ -37,7 +37,7 @@ func TestParseConfigCRLF(t *testing.T) {
 		t.Fatalf("Failed parsing config: %v", err)
 	}
 
-	cfg := ud.(CloudConfig)
+	cfg := ud.(*CloudConfig)
 
 	if cfg.Hostname != "foo" {
 		t.Error("Failed parsing hostname from config")
@@ -47,3 +47,12 @@ func TestParseConfigCRLF(t *testing.T) {
 		t.Error("Parsed incorrect number of SSH keys")
 	}
 }
+
+func TestParseConfigEmpty(t *testing.T) {
+	i, e := ParseUserData(``)
+	if i != nil {
+		t.Error("ParseUserData of empty string returned non-nil unexpectedly")
+	} else if e != nil {
+		t.Error("ParseUserData of empty string returned error unexpectedly")
+	}
+}

system/env_file.go

@@ -0,0 +1,89 @@
+package system
+
+import (
+	"bytes"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"path"
+	"regexp"
+)
+
+type EnvFile struct {
+	Vars map[string]string
+	// mask File.Content, it shouldn't be used.
+	Content interface{} `json:"-" yaml:"-"`
+	*File
+}
+
+// only allow sh compatible identifiers
+var validKey = regexp.MustCompile(`^[a-zA-Z0-9_]+$`)
+
+// match each line, optionally capturing valid identifiers, discarding dos line endings
+var lineLexer = regexp.MustCompile(`(?m)^((?:([a-zA-Z0-9_]+)=)?.*?)\r?\n`)
+
+// mergeEnvContents: Update the existing file contents with new values,
+// preserving variable ordering and all content this code doesn't understand.
+// All new values are appended to the bottom of the old.
+func mergeEnvContents(old []byte, pending map[string]string) []byte {
+	var buf bytes.Buffer
+	var match [][]byte
+
+	// it is awkward for the regex to handle a missing newline gracefully
+	if len(old) != 0 && !bytes.HasSuffix(old, []byte{'\n'}) {
+		old = append(old, byte('\n'))
+	}
+
+	for _, match = range lineLexer.FindAllSubmatch(old, -1) {
+		key := string(match[2])
+		if value, ok := pending[key]; ok {
+			fmt.Fprintf(&buf, "%s=%s\n", key, value)
+			delete(pending, key)
+		} else {
+			fmt.Fprintf(&buf, "%s\n", match[1])
+		}
+	}
+
+	for key, value := range pending {
+		fmt.Fprintf(&buf, "%s=%s\n", key, value)
+	}
+
+	return buf.Bytes()
+}
+
+// WriteEnvFile updates an existing env `KEY=value` formated file with
+// new values provided in EnvFile.Vars; File.Content is ignored.
+// Existing ordering and any unknown formatting such as comments are
+// preserved. If no changes are required the file is untouched.
+func WriteEnvFile(ef *EnvFile, root string) error {
+	// validate new keys, mergeEnvContents uses pending to track writes
+	pending := make(map[string]string, len(ef.Vars))
+	for key, value := range ef.Vars {
+		if !validKey.MatchString(key) {
+			return fmt.Errorf("Invalid name %q for %s", key, ef.Path)
+		}
+		pending[key] = value
+	}
+
+	if len(pending) == 0 {
+		return nil
+	}
+
+	oldContent, err := ioutil.ReadFile(path.Join(root, ef.Path))
+	if err != nil {
+		if os.IsNotExist(err) {
+			oldContent = []byte{}
+		} else {
+			return err
+		}
+	}
+
+	newContent := mergeEnvContents(oldContent, pending)
+	if bytes.Equal(oldContent, newContent) {
+		return nil
+	}
+
+	ef.File.Content = string(newContent)
+	_, err = WriteFile(ef.File, root)
+	return err
+}

system/env_file_test.go

@@ -0,0 +1,426 @@
+package system
+
+import (
+	"io/ioutil"
+	"os"
+	"path"
+	"strings"
+	"syscall"
+	"testing"
+)
+
+const (
+	base          = "# a file\nFOO=base\n\nBAR= hi there\n"
+	baseNoNewline = "# a file\nFOO=base\n\nBAR= hi there"
+	baseDos       = "# a file\r\nFOO=base\r\n\r\nBAR= hi there\r\n"
+	expectUpdate  = "# a file\nFOO=test\n\nBAR= hi there\nNEW=a value\n"
+	expectCreate  = "FOO=test\nNEW=a value\n"
+)
+
+var (
+	valueUpdate = map[string]string{
+		"FOO": "test",
+		"NEW": "a value",
+	}
+	valueNoop = map[string]string{
+		"FOO": "base",
+	}
+	valueEmpty   = map[string]string{}
+	valueInvalid = map[string]string{
+		"FOO-X": "test",
+	}
+)
+
+func TestWriteEnvFileUpdate(t *testing.T) {
+	dir, err := ioutil.TempDir(os.TempDir(), "coreos-cloudinit-")
+	if err != nil {
+		t.Fatalf("Unable to create tempdir: %v", err)
+	}
+	defer os.RemoveAll(dir)
+
+	name := "foo.conf"
+	fullPath := path.Join(dir, name)
+	ioutil.WriteFile(fullPath, []byte(base), 0644)
+
+	oldStat, err := os.Stat(fullPath)
+	if err != nil {
+		t.Fatal("Unable to stat file: %v", err)
+	}
+
+	ef := EnvFile{
+		File: &File{
+			Path: name,
+		},
+		Vars: valueUpdate,
+	}
+
+	err = WriteEnvFile(&ef, dir)
+	if err != nil {
+		t.Fatalf("WriteFile failed: %v", err)
+	}
+
+	contents, err := ioutil.ReadFile(fullPath)
+	if err != nil {
+		t.Fatalf("Unable to read expected file: %v", err)
+	}
+
+	if string(contents) != expectUpdate {
+		t.Fatalf("File has incorrect contents: %q", contents)
+	}
+
+	newStat, err := os.Stat(fullPath)
+	if err != nil {
+		t.Fatal("Unable to stat file: %v", err)
+	}
+
+	if oldStat.Sys().(*syscall.Stat_t).Ino == newStat.Sys().(*syscall.Stat_t).Ino {
+		t.Fatal("File was not replaced: %s", fullPath)
+	}
+}
+
+func TestWriteEnvFileUpdateNoNewline(t *testing.T) {
+	dir, err := ioutil.TempDir(os.TempDir(), "coreos-cloudinit-")
+	if err != nil {
+		t.Fatalf("Unable to create tempdir: %v", err)
+	}
+	defer os.RemoveAll(dir)
+
+	name := "foo.conf"
+	fullPath := path.Join(dir, name)
+	ioutil.WriteFile(fullPath, []byte(baseNoNewline), 0644)
+
+	oldStat, err := os.Stat(fullPath)
+	if err != nil {
+		t.Fatal("Unable to stat file: %v", err)
+	}
+
+	ef := EnvFile{
+		File: &File{
+			Path: name,
+		},
+		Vars: valueUpdate,
+	}
+
+	err = WriteEnvFile(&ef, dir)
+	if err != nil {
+		t.Fatalf("WriteFile failed: %v", err)
+	}
+
+	contents, err := ioutil.ReadFile(fullPath)
+	if err != nil {
+		t.Fatalf("Unable to read expected file: %v", err)
+	}
+
+	if string(contents) != expectUpdate {
+		t.Fatalf("File has incorrect contents: %q", contents)
+	}
+
+	newStat, err := os.Stat(fullPath)
+	if err != nil {
+		t.Fatal("Unable to stat file: %v", err)
+	}
+
+	if oldStat.Sys().(*syscall.Stat_t).Ino == newStat.Sys().(*syscall.Stat_t).Ino {
+		t.Fatal("File was not replaced: %s", fullPath)
+	}
+}
+
+func TestWriteEnvFileCreate(t *testing.T) {
+	dir, err := ioutil.TempDir(os.TempDir(), "coreos-cloudinit-")
+	if err != nil {
+		t.Fatalf("Unable to create tempdir: %v", err)
+	}
+	defer os.RemoveAll(dir)
+
+	name := "foo.conf"
+	fullPath := path.Join(dir, name)
+
+	ef := EnvFile{
+		File: &File{
+			Path: name,
+		},
+		Vars: valueUpdate,
+	}
+
+	err = WriteEnvFile(&ef, dir)
+	if err != nil {
+		t.Fatalf("WriteFile failed: %v", err)
+	}
+
+	contents, err := ioutil.ReadFile(fullPath)
+	if err != nil {
+		t.Fatalf("Unable to read expected file: %v", err)
+	}
+
+	if string(contents) != expectCreate {
+		t.Fatalf("File has incorrect contents: %q", contents)
+	}
+}
+
+func TestWriteEnvFileNoop(t *testing.T) {
+	dir, err := ioutil.TempDir(os.TempDir(), "coreos-cloudinit-")
+	if err != nil {
+		t.Fatalf("Unable to create tempdir: %v", err)
+	}
+	defer os.RemoveAll(dir)
+
+	name := "foo.conf"
+	fullPath := path.Join(dir, name)
+	ioutil.WriteFile(fullPath, []byte(base), 0644)
+
+	oldStat, err := os.Stat(fullPath)
+	if err != nil {
+		t.Fatal("Unable to stat file: %v", err)
+	}
+
+	ef := EnvFile{
+		File: &File{
+			Path: name,
+		},
+		Vars: valueNoop,
+	}
+
+	err = WriteEnvFile(&ef, dir)
+	if err != nil {
+		t.Fatalf("WriteFile failed: %v", err)
+	}
+
+	contents, err := ioutil.ReadFile(fullPath)
+	if err != nil {
+		t.Fatalf("Unable to read expected file: %v", err)
+	}
+
+	if string(contents) != base {
+		t.Fatalf("File has incorrect contents: %q", contents)
+	}
+
+	newStat, err := os.Stat(fullPath)
+	if err != nil {
+		t.Fatal("Unable to stat file: %v", err)
+	}
+
+	if oldStat.Sys().(*syscall.Stat_t).Ino != newStat.Sys().(*syscall.Stat_t).Ino {
+		t.Fatal("File was replaced: %s", fullPath)
+	}
+}
+
+func TestWriteEnvFileUpdateDos(t *testing.T) {
+	dir, err := ioutil.TempDir(os.TempDir(), "coreos-cloudinit-")
+	if err != nil {
+		t.Fatalf("Unable to create tempdir: %v", err)
+	}
+	defer os.RemoveAll(dir)
+
+	name := "foo.conf"
+	fullPath := path.Join(dir, name)
+	ioutil.WriteFile(fullPath, []byte(baseDos), 0644)
+
+	oldStat, err := os.Stat(fullPath)
+	if err != nil {
+		t.Fatal("Unable to stat file: %v", err)
+	}
+
+	ef := EnvFile{
+		File: &File{
+			Path: name,
+		},
+		Vars: valueUpdate,
+	}
+
+	err = WriteEnvFile(&ef, dir)
+	if err != nil {
+		t.Fatalf("WriteFile failed: %v", err)
+	}
+
+	contents, err := ioutil.ReadFile(fullPath)
+	if err != nil {
+		t.Fatalf("Unable to read expected file: %v", err)
+	}
+
+	if string(contents) != expectUpdate {
+		t.Fatalf("File has incorrect contents: %q", contents)
+	}
+
+	newStat, err := os.Stat(fullPath)
+	if err != nil {
+		t.Fatal("Unable to stat file: %v", err)
+	}
+
+	if oldStat.Sys().(*syscall.Stat_t).Ino == newStat.Sys().(*syscall.Stat_t).Ino {
+		t.Fatal("File was not replaced: %s", fullPath)
+	}
+}
+
+// A middle ground noop, values are unchanged but we did have a value.
+// Seems reasonable to rewrite the file in Unix format anyway.
+func TestWriteEnvFileDos2Unix(t *testing.T) {
+	dir, err := ioutil.TempDir(os.TempDir(), "coreos-cloudinit-")
+	if err != nil {
+		t.Fatalf("Unable to create tempdir: %v", err)
+	}
+	defer os.RemoveAll(dir)
+
+	name := "foo.conf"
+	fullPath := path.Join(dir, name)
+	ioutil.WriteFile(fullPath, []byte(baseDos), 0644)
+
+	oldStat, err := os.Stat(fullPath)
+	if err != nil {
+		t.Fatal("Unable to stat file: %v", err)
+	}
+
+	ef := EnvFile{
+		File: &File{
+			Path: name,
+		},
+		Vars: valueNoop,
+	}
+
+	err = WriteEnvFile(&ef, dir)
+	if err != nil {
+		t.Fatalf("WriteFile failed: %v", err)
+	}
+
+	contents, err := ioutil.ReadFile(fullPath)
+	if err != nil {
+		t.Fatalf("Unable to read expected file: %v", err)
+	}
+
+	if string(contents) != base {
+		t.Fatalf("File has incorrect contents: %q", contents)
+	}
+
+	newStat, err := os.Stat(fullPath)
+	if err != nil {
+		t.Fatal("Unable to stat file: %v", err)
+	}
+
+	if oldStat.Sys().(*syscall.Stat_t).Ino == newStat.Sys().(*syscall.Stat_t).Ino {
+		t.Fatal("File was not replaced: %s", fullPath)
+	}
+}
+
+// If it really is a noop (structure is empty) don't even do dos2unix
+func TestWriteEnvFileEmpty(t *testing.T) {
+	dir, err := ioutil.TempDir(os.TempDir(), "coreos-cloudinit-")
+	if err != nil {
+		t.Fatalf("Unable to create tempdir: %v", err)
+	}
+	defer os.RemoveAll(dir)
+
+	name := "foo.conf"
+	fullPath := path.Join(dir, name)
+	ioutil.WriteFile(fullPath, []byte(baseDos), 0644)
+
+	oldStat, err := os.Stat(fullPath)
+	if err != nil {
+		t.Fatal("Unable to stat file: %v", err)
+	}
+
+	ef := EnvFile{
+		File: &File{
+			Path: name,
+		},
+		Vars: valueEmpty,
+	}
+
+	err = WriteEnvFile(&ef, dir)
+	if err != nil {
+		t.Fatalf("WriteFile failed: %v", err)
+	}
+
+	contents, err := ioutil.ReadFile(fullPath)
+	if err != nil {
+		t.Fatalf("Unable to read expected file: %v", err)
+	}
+
+	if string(contents) != baseDos {
+		t.Fatalf("File has incorrect contents: %q", contents)
+	}
+
+	newStat, err := os.Stat(fullPath)
+	if err != nil {
+		t.Fatal("Unable to stat file: %v", err)
+	}
+
+	if oldStat.Sys().(*syscall.Stat_t).Ino != newStat.Sys().(*syscall.Stat_t).Ino {
+		t.Fatal("File was replaced: %s", fullPath)
+	}
+}
+
+// no point in creating empty files
+func TestWriteEnvFileEmptyNoCreate(t *testing.T) {
+	dir, err := ioutil.TempDir(os.TempDir(), "coreos-cloudinit-")
+	if err != nil {
+		t.Fatalf("Unable to create tempdir: %v", err)
+	}
+	defer os.RemoveAll(dir)
+
+	name := "foo.conf"
+	fullPath := path.Join(dir, name)
+
+	ef := EnvFile{
+		File: &File{
+			Path: name,
+		},
+		Vars: valueEmpty,
+	}
+
+	err = WriteEnvFile(&ef, dir)
+	if err != nil {
+		t.Fatalf("WriteFile failed: %v", err)
+	}
+
+	contents, err := ioutil.ReadFile(fullPath)
+	if err == nil {
+		t.Fatalf("File has incorrect contents: %q", contents)
+	} else if !os.IsNotExist(err) {
+		t.Fatalf("Unexpected error while reading file: %v", err)
+	}
+}
+
+func TestWriteEnvFilePermFailure(t *testing.T) {
+	dir, err := ioutil.TempDir(os.TempDir(), "coreos-cloudinit-")
+	if err != nil {
+		t.Fatalf("Unable to create tempdir: %v", err)
+	}
+	defer os.RemoveAll(dir)
+
+	name := "foo.conf"
+	fullPath := path.Join(dir, name)
+	ioutil.WriteFile(fullPath, []byte(base), 0000)
+
+	ef := EnvFile{
+		File: &File{
+			Path: name,
+		},
+		Vars: valueUpdate,
+	}
+
+	err = WriteEnvFile(&ef, dir)
+	if !os.IsPermission(err) {
+		t.Fatalf("Not a pemission denied error: %v", err)
+	}
+}
+
+func TestWriteEnvFileNameFailure(t *testing.T) {
+	dir, err := ioutil.TempDir(os.TempDir(), "coreos-cloudinit-")
+	if err != nil {
+		t.Fatalf("Unable to create tempdir: %v", err)
+	}
+	defer os.RemoveAll(dir)
+
+	name := "foo.conf"
+
+	ef := EnvFile{
+		File: &File{
+			Path: name,
+		},
+		Vars: valueInvalid,
+	}
+
+	err = WriteEnvFile(&ef, dir)
+	if err == nil || !strings.HasPrefix(err.Error(), "Invalid name") {
+		t.Fatalf("Not an invalid name error: %v", err)
+	}
+}

test

@@ -18,7 +18,7 @@ declare -a TESTPKGS=(initialize system datasource pkg network)
 if [ -z "$PKG" ]; then
 	GOFMTPATH="$TESTPKGS coreos-cloudinit.go"
 	# prepend repo path to each package
-	TESTPKGS=${TESTPKGS[@]/#/${REPO_PATH}/}
+	TESTPKGS="${TESTPKGS[@]/#/${REPO_PATH}/} ./"
 else
 	GOFMTPATH="$TESTPKGS"
 	# strip out slashes and dots from PKG=./foo/

units/user-configdrive.service

@@ -1,7 +1,7 @@
 [Unit]
 Description=Load cloud-config from /media/configdrive
 Requires=coreos-setup-environment.service
-After=coreos-setup-environment.service
+After=coreos-setup-environment.service system-config.target
 Before=user-config.target
 
 # HACK: work around ordering between config drive and ec2 metadata It is
@@ -13,7 +13,6 @@ Before=user-config.target
 # systemd knows about the ordering as early as possible.
 # coreos-cloudinit could implement a simple lock but that cannot be used
 # until after the systemd dbus calls are made non-blocking.
-After=system-cloudinit@usr-share-oem-cloud\x2dconfig.yml.service
 After=ec2-cloudinit.service
 
 [Service]