chore(release): Bump version to v0.2.2

doc(fields): Document field substitution

Documentation/cloud-config.md

@@ -8,29 +8,52 @@ Only a subset of [cloud-config functionality][cloud-config] is implemented. A se
 
 ## CoreOS Parameters
 
-### coreos.etcd.discovery_url
+### coreos.etcd
 
-The value of `coreos.etcd.discovery_url` will be used to discover the instance's etcd peers using the [etcd discovery protocol][disco-proto]. Usage of the [public discovery service][disco-service] is encouraged. **Note:** this is currently Amazon-only.
+The `coreos.etcd.*` options are translated to a partial systemd unit acting as an etcd configuration file.
+We can use the templating feature of coreos-cloudinit to automate etcd configuration with the `$private_ipv4` and `$public_ipv4` fields. For example, the following cloud-config document...
 
 ```
 #cloud-config
 
 coreos:
-  etcd:
+    etcd:
-    discovery_url: https://discovery.etcd.io/827c73219eeb2fa5530027c37bf18877
+        name: node001
+        discovery: https://discovery.etcd.io/3445fa65423d8b04df07f59fb40218f8
+        addr: $public_ipv4:4001
+        peer-addr: $private_ipv4:7001
 ```
 
-[disco-proto]: https://github.com/coreos/etcd/blob/master/Documentation/discovery-protocol.md
+...will generate a systemd unit drop-in like this:
-[disco-service]: http://discovery.etcd.io
+
+```
+[Service]
+Environment="ETCD_NAME=node001"
+Environment="ETCD_DISCOVERY=https://discovery.etcd.io/3445fa65423d8b04df07f59fb40218f8"
+Environment="ETCD_ADDR=203.0.113.29:4001"
+Environment="ETCD_PEER_ADDR=192.0.2.13:7001"
+```
+
+For more information about the available configuration options, see the [etcd documentation][etcd-config].
+Note that hyphens in the coreos.etcd.* keys are mapped to underscores.
+
+[etcd-config]: https://github.com/coreos/etcd/blob/master/Documentation/configuration.md
 
 ### coreos.units
 
 Arbitrary systemd units may be provided in the `coreos.units` attribute.
 `coreos.units` is a list of objects with the following fields:
 
-- **name**: string representing unit's name
+- **name**: String representing unit's name. Required.
-- **runtime**: boolean indicating whether or not to persist the unit across reboots. This is analagous to the `--runtime` flag to `systemd enable`.
+- **runtime**: Boolean indicating whether or not to persist the unit across reboots. This is analagous to the `--runtime` argument to `systemd enable`. Default value is false.
-- **content**: plaintext string representing entire unit file
+- **content**: Plaintext string representing entire unit file. If no value is provided, the unit is assumed to exist already.
+- **command**: Command to execute on unit: start, stop, reload, restart, try-restart, reload-or-restart, reload-or-try-restart. Default value is restart.
+
+**NOTE:** The command field is ignored for all network, netdev, and link units. The systemd-networkd.service unit will be restarted in their place.
+
+##### Examples
+
+Write a unit to disk, automatically starting it.
 
 ```
 #cloud-config
@@ -53,6 +76,19 @@ coreos:
           WantedBy=local.target
 ```
 
+Start the builtin `etcd` and `fleet` services:
+
+```
+# cloud-config
+
+coreos:
+    units:
+      - name: etcd.service
+        command: start
+      - name: fleet.service
+        command: start
+```
+
 ## Cloud-Config Parameters
 
 ### ssh_authorized_keys
@@ -149,13 +185,3 @@ Provide a list of objects with the following attributes:
 - **content**: Data to write at the provided `path`
 - **permissions**: String representing file permissions in octal notation (i.e. '0644')
 - **owner**: User and group that should own the file written to disk. This is equivalent to the `<user>:<group>` argument to `chown <user>:<group> <path>`.
-
-## user-data Script
-
-Simply set your user-data to a script where the first line is a shebang:
-
-```
-#!/bin/bash
-
-echo 'Hello, world!'
-```

README.md

@@ -1,9 +1,79 @@
 # coreos-cloudinit
 
-coreos-cloudinit enables a user to customize CoreOS machines by providing either an executable script or a cloud-config document as instance user-data.
+coreos-cloudinit enables a user to customize CoreOS machines by providing either a cloud-config document or an executable script through user-data.
 
-## Supported Cloud-Config Features
+## Configuration with cloud-config
 
-A subset of [cloud-config][cloud-config] is implemented in coreos-cloudinit and is [documented here](https://github.com/coreos/coreos-cloudinit/tree/master/Documentation/cloud-config.md). In addition specific CoreOS paramaters were added for unit files, etcd discovery urls, and others.
+A subset of the [official cloud-config spec][official-cloud-config] is implemented by coreos-cloudinit.
+Additionally, several [CoreOS-specific options][custom-cloud-config] have been implemented to support interacting with unit files, bootstrapping etcd clusters, and more.
+All supported cloud-config parameters are [documented here][all-cloud-config]. 
 
-[cloud-config]: http://cloudinit.readthedocs.org/en/latest/topics/format.html#cloud-config-data
+[official-cloud-config]: http://cloudinit.readthedocs.org/en/latest/topics/format.html#cloud-config-data
+[custom-cloud-config]: https://github.com/coreos/coreos-cloudinit/blob/master/Documentation/cloud-config.md#coreos-parameters
+[all-cloud-config]: https://github.com/coreos/coreos-cloudinit/tree/master/Documentation/cloud-config.md
+
+The following is an example cloud-config document:
+
+```
+#cloud-config
+
+coreos:
+    units:
+      - name: etcd.service
+        command: start
+
+users:
+  - name: core
+    passwd: $1$allJZawX$00S5T756I5PGdQga5qhqv1
+
+write_files:
+  - path: /etc/resolv.conf
+    content: |
+        nameserver 192.0.2.2
+        nameserver 192.0.2.3
+```
+
+## Executing a Script
+
+coreos-cloudinit supports executing user-data as a script instead of parsing it as a cloud-config document.
+Make sure the first line of your user-data is a shebang and coreos-cloudinit will attempt to execute it:
+
+```
+#!/bin/bash
+
+echo 'Hello, world!'
+```
+
+## user-data Field Substitution
+
+coreos-cloudinit will replace the following set of tokens in your user-data with system-generated values.
+
+| Token         | Description |
+| ------------- | ----------- |
+| $public_ipv4  | Public IPv4 address of machine |
+| $private_ipv4 | Private IPv4 address of machine |
+
+These values are determined by CoreOS based on the given provider on which your machine is running.
+Read more about provider-specific functionality in the [CoreOS OEM documentation][oem-doc].
+
+[oem-doc]: https://coreos.com/docs/sdk-distributors/distributors/notes-for-distributors/
+
+For example, submitting the following user-data...
+
+```
+#cloud-config
+coreos:
+    etcd:
+        addr: $public_ipv4:4001
+        peer-addr: $private_ipv4:7001
+```
+
+...will result in this cloud-config document being executed:
+
+```
+#cloud-config
+coreos:
+    etcd:
+        addr: 203.0.113.29:4001
+        peer-addr: 192.0.2.13:7001
+```

coreos-cloudinit.go

@@ -1,8 +1,6 @@
 package main
 
 import (
-	"bufio"
-	"bytes"
 	"flag"
 	"fmt"
 	"log"
@@ -14,7 +12,7 @@ import (
 	"github.com/coreos/coreos-cloudinit/system"
 )
 
-const version = "0.2.0"
+const version = "0.2.2"
 
 func main() {
 	var printVersion bool
@@ -58,7 +56,7 @@ func main() {
 	}
 
 	log.Printf("Fetching user-data from datasource of type %q", ds.Type())
-	userdata, err := ds.Fetch()
+	userdataBytes, err := ds.Fetch()
 	if err != nil {
 		log.Printf("Failed fetching user-data from datasource: %v", err)
 		if ignoreFailure {
@@ -68,11 +66,16 @@ func main() {
 		}
 	}
 
-	if len(userdata) == 0 {
+	if len(userdataBytes) == 0 {
 		log.Printf("No user data to handle, exiting.")
 		os.Exit(0)
 	}
 
+	env := initialize.NewEnvironment("/", workspace)
+
+	userdata := string(userdataBytes)
+	userdata = env.Apply(userdata)
+
 	parsed, err := ParseUserData(userdata)
 	if err != nil {
 		log.Printf("Failed parsing user-data: %v", err)
@@ -83,7 +86,6 @@ func main() {
 		}
 	}
 
-	env := initialize.NewEnvironment("/", workspace)
 	err = initialize.PrepWorkspace(env.Workspace())
 	if err != nil {
 		log.Fatalf("Failed preparing workspace: %v", err)
@@ -107,16 +109,14 @@ func main() {
 	}
 }
 
-func ParseUserData(contents []byte) (interface{}, error) {
+func ParseUserData(contents string) (interface{}, error) {
-	bytereader := bytes.NewReader(contents)
+	header := strings.SplitN(contents, "\n", 2)[0]
-	bufreader := bufio.NewReader(bytereader)
-	header, _ := bufreader.ReadString('\n')
 
 	if strings.HasPrefix(header, "#!") {
 		log.Printf("Parsing user-data as script")
 		return system.Script(contents), nil
 
-	} else if header == "#cloud-config\n" {
+	} else if header == "#cloud-config" {
 		log.Printf("Parsing user-data as cloud-config")
 		cfg, err := initialize.NewCloudConfig(contents)
 		if err != nil {

initialize/config.go

@@ -14,7 +14,6 @@ type CloudConfig struct {
 	SSHAuthorizedKeys []string `yaml:"ssh_authorized_keys"`
 	Coreos            struct {
 		Etcd  EtcdEnvironment
-		Fleet struct{ Autostart bool }
 		Units []system.Unit
 	}
 	WriteFiles []system.File `yaml:"write_files"`
@@ -22,9 +21,9 @@ type CloudConfig struct {
 	Users      []system.User
 }
 
-func NewCloudConfig(contents []byte) (*CloudConfig, error) {
+func NewCloudConfig(contents string) (*CloudConfig, error) {
 	var cfg CloudConfig
-	err := goyaml.Unmarshal(contents, &cfg)
+	err := goyaml.Unmarshal([]byte(contents), &cfg)
 	return &cfg, err
 }
 
@@ -115,34 +114,48 @@ func Apply(cfg CloudConfig, env *Environment) error {
 	}
 
 	if len(cfg.Coreos.Units) > 0 {
+		commands := make(map[string]string, 0)
+
 		for _, unit := range cfg.Coreos.Units {
-			log.Printf("Placing unit %s on filesystem", unit.Name)
+			if unit.Content != "" {
-			dst, err := system.PlaceUnit(&unit, env.Root())
+				log.Printf("Writing unit %s to filesystem", unit.Name)
+				dst, err := system.PlaceUnit(&unit, env.Root())
+				if err != nil {
+					return err
+				}
+				log.Printf("Placed unit %s at %s", unit.Name, dst)
+
+				if unit.Group() != "network" {
+					log.Printf("Enabling unit file %s", dst)
+					if err := system.EnableUnitFile(dst, unit.Runtime); err != nil {
+						return err
+					}
+					log.Printf("Enabled unit %s", unit.Name)
+				} else {
+					log.Printf("Skipping enable for network-like unit %s", unit.Name)
+				}
+			}
+
+			if unit.Group() != "network" {
+				command := unit.Command
+				if command == "" {
+					command = "restart"
+				}
+				commands[unit.Name] = command
+			} else {
+				commands["systemd-networkd.service"] = "restart"
+			}
+		}
+
+		system.DaemonReload()
+
+		for unit, command := range commands {
+			log.Printf("Calling unit command '%s %s'", command, unit)
+			res, err := system.RunUnitCommand(command, unit)
 			if err != nil {
 				return err
 			}
-			log.Printf("Placed unit %s at %s", unit.Name, dst)
+			log.Printf("Result of '%s %s': %s", command, unit, res)
-
-			if unit.Group() != "network" {
-				log.Printf("Enabling unit file %s", dst)
-				if err := system.EnableUnitFile(dst, unit.Runtime); err != nil {
-					return err
-				}
-				log.Printf("Enabled unit %s", unit.Name)
-			} else {
-				log.Printf("Skipping enable for network-like unit %s", unit.Name)
-			}
-		}
-		system.DaemonReload()
-		system.StartUnits(cfg.Coreos.Units)
-	}
-
-	if cfg.Coreos.Fleet.Autostart {
-		err := system.StartUnitByName("fleet.service")
-		if err == nil {
-			log.Printf("Started fleet service.")
-		} else {
-			return err
 		}
 	}
 

initialize/config_test.go

@@ -7,7 +7,7 @@ import (
 
 // Assert that the parsing of a cloud config file "generally works"
 func TestCloudConfigEmpty(t *testing.T) {
-	cfg, err := NewCloudConfig([]byte{})
+	cfg, err := NewCloudConfig("")
 	if err != nil {
 		t.Fatalf("Encountered unexpected error :%v", err)
 	}
@@ -17,10 +17,6 @@ func TestCloudConfigEmpty(t *testing.T) {
 		t.Error("Parsed incorrect number of SSH keys")
 	}
 
-	if cfg.Coreos.Fleet.Autostart {
-		t.Error("Expected AutostartFleet not to be defined")
-	}
-
 	if len(cfg.WriteFiles) != 0 {
 		t.Error("Expected zero WriteFiles")
 	}
@@ -32,12 +28,10 @@ func TestCloudConfigEmpty(t *testing.T) {
 
 // Assert that the parsing of a cloud config file "generally works"
 func TestCloudConfig(t *testing.T) {
-	contents := []byte(`
+	contents := `
 coreos: 
   etcd:
     discovery: "https://discovery.etcd.io/827c73219eeb2fa5530027c37bf18877"
-  fleet:
-    autostart: Yes
   units:
     - name: 50-eth0.network
       runtime: yes
@@ -62,7 +56,7 @@ write_files:
     permissions: '0644'
     owner: root:dogepack
 hostname: trontastic
-`)
+`
 	cfg, err := NewCloudConfig(contents)
 	if err != nil {
 		t.Fatalf("Encountered unexpected error :%v", err)
@@ -77,10 +71,6 @@ hostname: trontastic
 		t.Error("Expected first SSH key to be 'foobaz'")
 	}
 
-	if !cfg.Coreos.Fleet.Autostart {
-		t.Error("Expected AutostartFleet to be true")
-	}
-
 	if len(cfg.WriteFiles) != 1 {
 		t.Error("Failed to parse correct number of write_files")
 	} else {
@@ -133,10 +123,10 @@ Address=10.209.171.177/19
 
 // Assert that our interface conversion doesn't panic
 func TestCloudConfigKeysNotList(t *testing.T) {
-	contents := []byte(`
+	contents := `
 ssh_authorized_keys:
   - foo: bar
-`)
+`
 	cfg, err := NewCloudConfig(contents)
 	if err != nil {
 		t.Fatalf("Encountered unexpected error :%v", err)
@@ -149,7 +139,7 @@ ssh_authorized_keys:
 }
 
 func TestCloudConfigSerializationHeader(t *testing.T) {
-	cfg, _ := NewCloudConfig([]byte{})
+	cfg, _ := NewCloudConfig("")
 	contents := cfg.String()
 	header := strings.SplitN(contents, "\n", 2)[0]
 	if header != "#cloud-config" {
@@ -158,7 +148,7 @@ func TestCloudConfigSerializationHeader(t *testing.T) {
 }
 
 func TestCloudConfigUsers(t *testing.T) {
-	contents := []byte(`
+	contents := `
 users:
   - name: elroy
     passwd: somehash
@@ -174,7 +164,7 @@ users:
     no-user-group: true
     system: y
     no-log-init: True
-`)
+`
 	cfg, err := NewCloudConfig(contents)
 	if err != nil {
 		t.Fatalf("Encountered unexpected error: %v", err)

initialize/env.go

@@ -1,19 +1,26 @@
 package initialize
 
 import (
+	"os"
 	"path"
+	"strings"
 )
 
 const DefaultSSHKeyName = "coreos-cloudinit"
 
 type Environment struct {
-	root      string
+	root          string
-	workspace string
+	workspace     string
-	sshKeyName string
+	sshKeyName    string
+	substitutions map[string]string
 }
 
 func NewEnvironment(root, workspace string) *Environment {
-	return &Environment{root, workspace, DefaultSSHKeyName}
+	substitutions := map[string]string{
+		"$public_ipv4":  os.Getenv("COREOS_PUBLIC_IPV4"),
+		"$private_ipv4": os.Getenv("COREOS_PRIVATE_IPV4"),
+	}
+	return &Environment{root, workspace, DefaultSSHKeyName, substitutions}
 }
 
 func (self *Environment) Workspace() string {
@@ -31,3 +38,10 @@ func (self *Environment) SSHKeyName() string {
 func (self *Environment) SetSSHKeyName(name string) {
 	self.sshKeyName = name
 }
+
+func (self *Environment) Apply(data string) string {
+	for key, val := range self.substitutions {
+		data = strings.Replace(data, key, val, -1)
+	}
+	return data
+}

initialize/env_test.go

@@ -0,0 +1,27 @@
+package initialize
+
+import (
+	"os"
+	"testing"
+)
+
+func TestEnvironmentApply(t *testing.T) {
+	os.Setenv("COREOS_PUBLIC_IPV4", "192.0.2.3")
+	os.Setenv("COREOS_PRIVATE_IPV4", "192.0.2.203")
+	env := NewEnvironment("./", "./")
+	input := `[Service]
+ExecStart=/usr/bin/echo "$public_ipv4"
+ExecStop=/usr/bin/echo $private_ipv4
+ExecStop=/usr/bin/echo $unknown
+`
+	expected := `[Service]
+ExecStart=/usr/bin/echo "192.0.2.3"
+ExecStop=/usr/bin/echo 192.0.2.203
+ExecStop=/usr/bin/echo $unknown
+`
+
+	output := env.Apply(input)
+	if output != expected {
+		t.Fatalf("Environment incorrectly applied.\nOutput:\n%s\nExpected:\n%s", output, expected)
+	}
+}

initialize/etcd.go

@@ -3,7 +3,6 @@ package initialize
 import (
 	"errors"
 	"fmt"
-	"os"
 	"path"
 	"strings"
 
@@ -32,20 +31,9 @@ func (ec EtcdEnvironment) String() (out string) {
 		}
 	}
 
-	public := os.Getenv("COREOS_PUBLIC_IPV4")
-	private := os.Getenv("COREOS_PRIVATE_IPV4")
-
 	out += "[Service]\n"
 
 	for key, val := range norm {
-		if public != "" {
-			val = strings.Replace(val, "$public_ipv4", public, -1)
-		}
-
-		if private != "" {
-			val = strings.Replace(val, "$private_ipv4", private, -1)
-		}
-
 		out += fmt.Sprintf("Environment=\"ETCD_%s=%s\"\n", key, val)
 	}
 

initialize/etcd_test.go

@@ -58,25 +58,6 @@ Environment="ETCD_PEER_BIND_ADDR=127.0.0.1:7002"
 	}
 }
 
-func TestEtcdEnvironmentReplacement(t *testing.T) {
-	os.Clearenv()
-	os.Setenv("COREOS_PUBLIC_IPV4", "203.0.113.29")
-	os.Setenv("COREOS_PRIVATE_IPV4", "192.0.2.13")
-
-	cfg := make(EtcdEnvironment, 0)
-	cfg["bind-addr"] = "$public_ipv4:4001"
-	cfg["peer-bind-addr"] = "$private_ipv4:7001"
-
-	env := cfg.String()
-	expect := `[Service]
-Environment="ETCD_BIND_ADDR=203.0.113.29:4001"
-Environment="ETCD_PEER_BIND_ADDR=192.0.2.13:7001"
-`
-	if env != expect {
-		t.Errorf("Generated environment:\n%s\nExpected environment:\n%s", env, expect)
-	}
-}
-
 func TestEtcdEnvironmentWrittenToDisk(t *testing.T) {
 	ec := EtcdEnvironment{
 		"name": "node001",

initialize/github_test.go

@@ -45,11 +45,11 @@ func TestCloudConfigUsersGithubMarshal(t *testing.T) {
 }
 func TestCloudConfigUsersGithubUser(t *testing.T) {
 
-	contents := []byte(`
+	contents := `
 users:
   - name: elroy
     coreos-ssh-import-github: bcwaldon
-`)
+`
 	cfg, err := NewCloudConfig(contents)
 	if err != nil {
 		t.Fatalf("Encountered unexpected error: %v", err)

system/systemd.go

@@ -21,6 +21,7 @@ type Unit struct {
 	Name    string
 	Runtime bool
 	Content string
+	Command string
 }
 
 func (u *Unit) Type() string {
@@ -80,34 +81,33 @@ func EnableUnitFile(file string, runtime bool) error {
 	return err
 }
 
-func separateNetworkUnits(units []Unit) ([]Unit, []Unit) {
+func RunUnitCommand(command, unit string) (string, error) {
-	networkUnits := make([]Unit, 0)
+	conn, err := dbus.New()
-	nonNetworkUnits := make([]Unit, 0)
+	if err != nil {
-	for _, unit := range units {
+		return "", err
-		if unit.Group() == "network" {
-			networkUnits = append(networkUnits, unit)
-		} else {
-			nonNetworkUnits = append(nonNetworkUnits, unit)
-		}
-	}
-	return networkUnits, nonNetworkUnits
-}
-
-func StartUnits(units []Unit) error {
-	networkUnits, nonNetworkUnits := separateNetworkUnits(units)
-	if len(networkUnits) > 0 {
-		if err := RestartUnitByName("systemd-networkd.service"); err != nil {
-			return err
-		}
 	}
 
-	for _, unit := range nonNetworkUnits {
+	var fn func(string, string) (string, error)
-		if err := RestartUnitByName(unit.Name); err != nil {
+	switch command {
-			return err
+	case "start":
-		}
+		fn = conn.StartUnit
+	case "stop":
+		fn = conn.StopUnit
+	case "restart":
+		fn = conn.RestartUnit
+	case "reload":
+		fn = conn.ReloadUnit
+	case "try-restart":
+		fn = conn.TryRestartUnit
+	case "reload-or-restart":
+		fn = conn.ReloadOrRestartUnit
+	case "reload-or-try-restart":
+		fn = conn.ReloadOrTryRestartUnit
+	default:
+		return "", fmt.Errorf("Unsupported systemd command %q", command)
 	}
 
-	return nil
+	return fn(unit, "replace")
 }
 
 func DaemonReload() error {
@@ -120,29 +120,6 @@ func DaemonReload() error {
 	return err
 }
 
-func RestartUnitByName(name string) error {
-	log.Printf("Restarting unit %s", name)
-	conn, err := dbus.New()
-	if err != nil {
-		return err
-	}
-
-	output, err := conn.RestartUnit(name, "replace")
-	log.Printf("Restart completed with '%s'", output)
-
-	return err
-}
-
-func StartUnitByName(name string) error {
-	conn, err := dbus.New()
-	if err != nil {
-		return err
-	}
-
-	_, err = conn.StartUnit(name, "replace")
-	return err
-}
-
 func ExecuteScript(scriptPath string) (string, error) {
 	props := []dbus.Property{
 		dbus.PropDescription("Unit generated and executed by coreos-cloudinit on behalf of user"),