chore(coreos-cloudinit): bump to 0.7.2

"Enable" option does not support units in /usr/lib64/systemd

.gitignore

@@ -1,4 +1,4 @@
 *.swp
 bin/
 coverage/
-pkg/
+gopath/

Documentation/cloud-config.md

@@ -97,11 +97,15 @@ For more information on fleet configuration, see the [fleet documentation][fleet
 
 The `coreos.update.*` parameters manipulate settings related to how CoreOS instances are updated.
 
+These fields will be written out to and replace `/etc/coreos/update.conf`. If only one of the parameters is given it will only overwrite the given field.
+
 - **reboot-strategy**: One of "reboot", "etcd-lock", "best-effort" or "off" for controlling when reboots are issued after an update is performed.
   - _reboot_: Reboot immediately after an update is applied.
   - _etcd-lock_: Reboot after first taking a distributed lock in etcd, this guarantees that only one host will reboot concurrently and that the cluster will remain available during the update.
   - _best-effort_ - If etcd is running, "etcd-lock", otherwise simply "reboot".
   - _off_ - Disable rebooting after updates are applied (not recommended).
+- **server**: is the omaha endpoint URL which will be queried for updates.
+- **group**:  signifies the channel which should be used for automatic updates.  This value defaults to the version of the image initially downloaded. (one of "master", "alpha", "beta", "stable")
 
 ```
 #cloud-config

build

@@ -3,7 +3,12 @@
 ORG_PATH="github.com/coreos"
 REPO_PATH="${ORG_PATH}/coreos-cloudinit"
 
+if [ ! -h gopath/src/${REPO_PATH} ]; then
+	mkdir -p gopath/src/${ORG_PATH}
+	ln -s ../../../.. gopath/src/${REPO_PATH} || exit 255
+fi
+
 export GOBIN=${PWD}/bin
-export GOPATH=${PWD}
+export GOPATH=${PWD}/gopath
 
 go build -o bin/coreos-cloudinit ${REPO_PATH}

coreos-cloudinit.go

@@ -11,7 +11,12 @@ import (
 	"github.com/coreos/coreos-cloudinit/system"
 )
 
-const version = "0.7.0"
+const version = "0.7.2"
+
+func init() {
+	//Removes timestamp since it is displayed already during booting
+	log.SetFlags(0)
+}
 
 func main() {
 	var printVersion bool

datasource/datasource.go

@@ -1,104 +1,6 @@
 package datasource
 
-import (
-	"errors"
-	"fmt"
-	"io/ioutil"
-	"log"
-	"math"
-	"net"
-	"net/http"
-	neturl "net/url"
-	"strings"
-	"time"
-)
-
-const (
-	HTTP_2xx = 2
-	HTTP_4xx = 4
-
-	maxTimeout = time.Second * 5
-	maxRetries = 15
-)
-
 type Datasource interface {
 	Fetch() ([]byte, error)
 	Type() string
 }
-
-// HTTP client timeout
-// This one is low since exponential backoff will kick off too.
-var timeout = time.Duration(2) * time.Second
-
-func dialTimeout(network, addr string) (net.Conn, error) {
-	deadline := time.Now().Add(timeout)
-	c, err := net.DialTimeout(network, addr, timeout)
-	if err != nil {
-		return nil, err
-	}
-	c.SetDeadline(deadline)
-	return c, nil
-}
-
-// Fetches user-data url with support for exponential backoff and maximum retries
-func fetchURL(rawurl string) ([]byte, error) {
-	if rawurl == "" {
-		return nil, errors.New("user-data URL is empty. Skipping.")
-	}
-
-	url, err := neturl.Parse(rawurl)
-	if err != nil {
-		return nil, err
-	}
-
-	// Unfortunately, url.Parse is too generic to throw errors if a URL does not
-	// have a valid HTTP scheme. So, we have to do this extra validation
-	if !strings.HasPrefix(url.Scheme, "http") {
-		return nil, fmt.Errorf("user-data URL %s does not have a valid HTTP scheme. Skipping.", rawurl)
-	}
-
-	userdataURL := url.String()
-
-	// We need to create our own client in order to add timeout support.
-	// TODO(c4milo) Replace it once Go 1.3 is officially used by CoreOS
-	// More info: https://code.google.com/p/go/source/detail?r=ada6f2d5f99f
-	transport := &http.Transport{
-		Dial: dialTimeout,
-	}
-
-	client := &http.Client{
-		Transport: transport,
-	}
-
-	for retry := 1; retry <= maxRetries; retry++ {
-		log.Printf("Fetching user-data from %s. Attempt #%d", userdataURL, retry)
-
-		resp, err := client.Get(userdataURL)
-
-		if err == nil {
-			defer resp.Body.Close()
-			status := resp.StatusCode / 100
-
-			if status == HTTP_2xx {
-				return ioutil.ReadAll(resp.Body)
-			}
-
-			if status == HTTP_4xx {
-				return nil, fmt.Errorf("user-data not found. HTTP status code: %d", resp.StatusCode)
-			}
-
-			log.Printf("user-data not found. HTTP status code: %d", resp.StatusCode)
-		} else {
-			log.Printf("unable to fetch user-data: %s", err.Error())
-		}
-
-		duration := time.Millisecond * time.Duration((math.Pow(float64(2), float64(retry)) * 100))
-		if duration > maxTimeout {
-			duration = maxTimeout
-		}
-
-		time.Sleep(duration)
-	}
-
-	return nil, fmt.Errorf("unable to fetch user-data. Maximum retries reached: %d", maxRetries)
-}

datasource/metadata_service.go

@@ -1,5 +1,7 @@
 package datasource
 
+import "github.com/coreos/coreos-cloudinit/pkg"
+
 type metadataService struct {
 	url string
 }
@@ -9,7 +11,8 @@ func NewMetadataService(url string) *metadataService {
 }
 
 func (ms *metadataService) Fetch() ([]byte, error) {
-	return fetchURL(ms.url)
+	client := pkg.NewHttpClient()
+	return client.Get(ms.url)
 }
 
 func (ms *metadataService) Type() string {

datasource/proc_cmdline.go

@@ -5,6 +5,8 @@ import (
 	"io/ioutil"
 	"log"
 	"strings"
+
+	"github.com/coreos/coreos-cloudinit/pkg"
 )
 
 const (
@@ -12,24 +14,28 @@ const (
 	ProcCmdlineCloudConfigFlag = "cloud-config-url"
 )
 
-type procCmdline struct{}
+type procCmdline struct{
+	Location string
+}
 
 func NewProcCmdline() *procCmdline {
-	return &procCmdline{}
+	return &procCmdline{Location: ProcCmdlineLocation}
 }
 
 func (self *procCmdline) Fetch() ([]byte, error) {
-	cmdline, err := ioutil.ReadFile(ProcCmdlineLocation)
+	contents, err := ioutil.ReadFile(self.Location)
 	if err != nil {
 		return nil, err
 	}
 
-	url, err := findCloudConfigURL(string(cmdline))
+	cmdline := strings.TrimSpace(string(contents))
+	url, err := findCloudConfigURL(cmdline)
 	if err != nil {
 		return nil, err
 	}
 
-	cfg, err := fetchURL(url)
+	client := pkg.NewHttpClient()
+	cfg, err := client.Get(url)
 	if err != nil {
 		return nil, err
 	}

datasource/proc_cmdline_test.go

@@ -1,6 +1,11 @@
 package datasource
 
 import (
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
+	"os"
 	"testing"
 )
 
@@ -45,3 +50,39 @@ func TestParseCmdlineCloudConfigFound(t *testing.T) {
 		}
 	}
 }
+
+func TestProcCmdlineAndFetchConfig(t *testing.T) {
+
+	var (
+		ProcCmdlineTmpl    = "foo=bar cloud-config-url=%s/config\n"
+		CloudConfigContent = "#cloud-config\n"
+	)
+
+	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.Method == "GET" && r.RequestURI == "/config" {
+			fmt.Fprint(w, CloudConfigContent)
+		}
+	}))
+	defer ts.Close()
+
+	file, err := ioutil.TempFile(os.TempDir(), "test_proc_cmdline")
+	defer os.Remove(file.Name())
+	if err != nil {
+		t.Errorf("Test produced error: %v", err)
+	}
+	_, err = file.Write([]byte(fmt.Sprintf(ProcCmdlineTmpl, ts.URL)))
+	if err != nil {
+		t.Errorf("Test produced error: %v", err)
+	}
+
+	p := NewProcCmdline()
+	p.Location = file.Name()
+	cfg, err := p.Fetch()
+	if err != nil {
+		t.Errorf("Test produced error: %v", err)
+	}
+
+	if string(cfg) != CloudConfigContent {
+		t.Errorf("Test failed, response body: %s != %s", cfg, CloudConfigContent)
+	}
+}

initialize/config.go

@@ -254,8 +254,8 @@ func Apply(cfg CloudConfig, env *Environment) error {
 
 		if unit.Enable {
 			if unit.Group() != "network" {
-				log.Printf("Enabling unit file %s", dst)
-				if err := system.EnableUnitFile(dst, unit.Runtime); err != nil {
+				log.Printf("Enabling unit file %s", unit.Name)
+				if err := system.EnableUnitFile(unit.Name, unit.Runtime); err != nil {
 					return err
 				}
 				log.Printf("Enabled unit %s", unit.Name)

initialize/etcd.go

@@ -31,6 +31,10 @@ func (ee EtcdEnvironment) String() (out string) {
 // Unit creates a Unit file drop-in for etcd, using any configured
 // options and adding a default MachineID if unset.
 func (ee EtcdEnvironment) Unit(root string) (*system.Unit, error) {
+	if ee == nil {
+		return nil, nil
+	}
+
 	if _, ok := ee["name"]; !ok {
 		if machineID := system.MachineID(root); machineID != "" {
 			ee["name"] = machineID

initialize/etcd_test.go

@@ -152,3 +152,15 @@ Environment="ETCD_NAME=node007"
 		t.Fatalf("File has incorrect contents")
 	}
 }
+
+func TestEtcdEnvironmentWhenNil(t *testing.T) {
+	// EtcdEnvironment will be a nil map if it wasn't in the yaml
+	var ee EtcdEnvironment
+	if ee != nil {
+		t.Fatalf("EtcdEnvironment is not nil")
+	}
+	u, err := ee.Unit("")
+	if u != nil || err != nil {
+		t.Fatalf("Unit returned a non-nil value for nil input")
+	}
+}

initialize/ssh_keys.go

@@ -3,9 +3,8 @@ package initialize
 import (
 	"encoding/json"
 	"fmt"
-	"io/ioutil"
-	"net/http"
 
+	"github.com/coreos/coreos-cloudinit/pkg"
 	"github.com/coreos/coreos-cloudinit/system"
 )
 
@@ -25,22 +24,19 @@ func SSHImportKeysFromURL(system_user string, url string) error {
 }
 
 func fetchUserKeys(url string) ([]string, error) {
-	res, err := http.Get(url)
+	client := pkg.NewHttpClient()
+	data, err := client.Get(url)
 	if err != nil {
 		return nil, err
 	}
-	defer res.Body.Close()
-	body, err := ioutil.ReadAll(res.Body)
-	if err != nil {
-		return nil, err
-	}
-	var data []UserKey
-	err = json.Unmarshal(body, &data)
+
+	var userKeys []UserKey
+	err = json.Unmarshal(data, &userKeys)
 	if err != nil {
 		return nil, err
 	}
 	keys := make([]string, 0)
-	for _, key := range data {
+	for _, key := range userKeys {
 		keys = append(keys, key.Key)
 	}
 	return keys, err

initialize/update.go

@@ -136,13 +136,11 @@ func (uc UpdateConfig) Unit(root string) (*system.Unit, error) {
 
 	u := &system.Unit{
 		Name:    locksmithUnit,
-		Enable:  true,
 		Command: "restart",
 		Mask:    false,
 	}
 
 	if strategy == "off" {
-		u.Enable = false
 		u.Command = "stop"
 		u.Mask = true
 	}

pkg/http_client.go

@@ -0,0 +1,120 @@
+package pkg
+
+import (
+	"crypto/tls"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"math"
+	"net"
+	"net/http"
+	neturl "net/url"
+	"strings"
+	"time"
+)
+
+const (
+	HTTP_2xx = 2
+	HTTP_4xx = 4
+)
+
+type HttpClient struct {
+	// Maximum exp backoff duration. Defaults to 5 seconds
+	MaxBackoff time.Duration
+
+	// Maximum number of connection retries. Defaults to 15
+	MaxRetries int
+
+	// HTTP client timeout, this is suggested to be low since exponential
+	// backoff will kick off too. Defaults to 2 seconds
+	Timeout time.Duration
+
+	// Whether or not to skip TLS verification. Defaults to false
+	SkipTLS bool
+}
+
+func NewHttpClient() *HttpClient {
+	return &HttpClient{
+		MaxBackoff: time.Second * 5,
+		MaxRetries: 15,
+		Timeout:    time.Duration(2) * time.Second,
+		SkipTLS:    false,
+	}
+}
+
+// Fetches a given URL with support for exponential backoff and maximum retries
+func (h *HttpClient) Get(rawurl string) ([]byte, error) {
+	if rawurl == "" {
+		return nil, errors.New("URL is empty. Skipping.")
+	}
+
+	url, err := neturl.Parse(rawurl)
+	if err != nil {
+		return nil, err
+	}
+
+	// Unfortunately, url.Parse is too generic to throw errors if a URL does not
+	// have a valid HTTP scheme. So, we have to do this extra validation
+	if !strings.HasPrefix(url.Scheme, "http") {
+		return nil, fmt.Errorf("URL %s does not have a valid HTTP scheme. Skipping.", rawurl)
+	}
+
+	dataURL := url.String()
+
+	// We need to create our own client in order to add timeout support.
+	// TODO(c4milo) Replace it once Go 1.3 is officially used by CoreOS
+	// More info: https://code.google.com/p/go/source/detail?r=ada6f2d5f99f
+	transport := &http.Transport{
+		TLSClientConfig: &tls.Config{
+			InsecureSkipVerify: h.SkipTLS,
+		},
+		Dial: func(network, addr string) (net.Conn, error) {
+			deadline := time.Now().Add(h.Timeout)
+			c, err := net.DialTimeout(network, addr, h.Timeout)
+			if err != nil {
+				return nil, err
+			}
+			c.SetDeadline(deadline)
+			return c, nil
+		},
+	}
+
+	client := &http.Client{
+		Transport: transport,
+	}
+
+	for retry := 1; retry <= h.MaxRetries; retry++ {
+		log.Printf("Fetching data from %s. Attempt #%d", dataURL, retry)
+
+		resp, err := client.Get(dataURL)
+
+		if err == nil {
+			defer resp.Body.Close()
+			status := resp.StatusCode / 100
+
+			if status == HTTP_2xx {
+				return ioutil.ReadAll(resp.Body)
+			}
+
+			if status == HTTP_4xx {
+				return nil, fmt.Errorf("Not found. HTTP status code: %d", resp.StatusCode)
+			}
+
+			log.Printf("Server error. HTTP status code: %d", resp.StatusCode)
+		} else {
+			log.Printf("Unable to fetch data: %s", err.Error())
+		}
+
+		duration := time.Millisecond * time.Duration((math.Pow(float64(2), float64(retry)) * 100))
+		if duration > h.MaxBackoff {
+			duration = h.MaxBackoff
+		}
+
+		log.Printf("Sleeping for %v...", duration)
+
+		time.Sleep(duration)
+	}
+
+	return nil, fmt.Errorf("Unable to fetch data. Maximum retries reached: %d", h.MaxRetries)
+}

pkg/http_client_test.go

@@ -1,4 +1,4 @@
-package datasource
+package pkg
 
 import (
 	"fmt"
@@ -19,7 +19,9 @@ var expBackoffTests = []struct {
 
 // Test exponential backoff and that it continues retrying if a 5xx response is
 // received
-func TestFetchURLExpBackOff(t *testing.T) {
+func TestGetURLExpBackOff(t *testing.T) {
+	client := NewHttpClient()
+
 	for i, tt := range expBackoffTests {
 		mux := http.NewServeMux()
 		count := 0
@@ -34,7 +36,7 @@ func TestFetchURLExpBackOff(t *testing.T) {
 		ts := httptest.NewServer(mux)
 		defer ts.Close()
 
-		data, err := fetchURL(ts.URL)
+		data, err := client.Get(ts.URL)
 		if err != nil {
 			t.Errorf("Test case %d produced error: %v", i, err)
 		}
@@ -50,7 +52,8 @@ func TestFetchURLExpBackOff(t *testing.T) {
 }
 
 // Test that it stops retrying if a 4xx response comes back
-func TestFetchURL4xx(t *testing.T) {
+func TestGetURL4xx(t *testing.T) {
+	client := NewHttpClient()
 	retries := 0
 	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		retries++
@@ -58,9 +61,9 @@ func TestFetchURL4xx(t *testing.T) {
 	}))
 	defer ts.Close()
 
-	_, err := fetchURL(ts.URL)
+	_, err := client.Get(ts.URL)
 	if err == nil {
-		t.Errorf("Incorrect result\ngot:  %s\nwant: %s", err.Error(), "user-data not found. HTTP status code: 404")
+		t.Errorf("Incorrect result\ngot:  %s\nwant: %s", err.Error(), "Not found. HTTP status code: 404")
 	}
 
 	if retries > 1 {
@@ -69,7 +72,7 @@ func TestFetchURL4xx(t *testing.T) {
 }
 
 // Test that it fetches and returns user-data just fine
-func TestFetchURL2xx(t *testing.T) {
+func TestGetURL2xx(t *testing.T) {
 	var cloudcfg = `
 #cloud-config
 coreos: 
@@ -83,12 +86,13 @@ coreos:
 		reboot-strategy: best-effort
 `
 
+	client := NewHttpClient()
 	ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
 		fmt.Fprint(w, cloudcfg)
 	}))
 	defer ts.Close()
 
-	data, err := fetchURL(ts.URL)
+	data, err := client.Get(ts.URL)
 	if err != nil {
 		t.Errorf("Incorrect result\ngot:  %v\nwant: %v", err, nil)
 	}
@@ -99,19 +103,21 @@ coreos:
 }
 
 // Test attempt to fetching using malformed URL
-func TestFetchURLMalformed(t *testing.T) {
+func TestGetMalformedURL(t *testing.T) {
+	client := NewHttpClient()
+
 	var tests = []struct {
 		url  string
 		want string
 	}{
-		{"boo", "user-data URL boo does not have a valid HTTP scheme. Skipping."},
-		{"mailto://boo", "user-data URL mailto://boo does not have a valid HTTP scheme. Skipping."},
-		{"ftp://boo", "user-data URL ftp://boo does not have a valid HTTP scheme. Skipping."},
-		{"", "user-data URL is empty. Skipping."},
+		{"boo", "URL boo does not have a valid HTTP scheme. Skipping."},
+		{"mailto://boo", "URL mailto://boo does not have a valid HTTP scheme. Skipping."},
+		{"ftp://boo", "URL ftp://boo does not have a valid HTTP scheme. Skipping."},
+		{"", "URL is empty. Skipping."},
 	}
 
 	for _, test := range tests {
-		_, err := fetchURL(test.url)
+		_, err := client.Get(test.url)
 		if err == nil || err.Error() != test.want {
 			t.Errorf("Incorrect result\ngot:  %v\nwant: %v", err, test.want)
 		}

src/github.com/coreos/coreos-cloudinit

@@ -1 +0,0 @@
-../../../

system/systemd.go

@@ -91,14 +91,14 @@ func PlaceUnit(u *Unit, dst string) error {
 	return nil
 }
 
-func EnableUnitFile(file string, runtime bool) error {
+func EnableUnitFile(unit string, runtime bool) error {
 	conn, err := dbus.New()
 	if err != nil {
 		return err
 	}
 
-	files := []string{file}
-	_, _, err = conn.EnableUnitFiles(files, runtime, true)
+	units := []string{unit}
+	_, _, err = conn.EnableUnitFiles(units, runtime, true)
 	return err
 }
 

test

@@ -13,7 +13,7 @@ COVER=${COVER:-"-cover"}
 
 source ./build
 
-declare -a TESTPKGS=(initialize system datasource)
+declare -a TESTPKGS=(initialize system datasource pkg)
 
 if [ -z "$PKG" ]; then
 	GOFMTPATH="$TESTPKGS coreos-cloudinit.go"

units/configdrive-block.service

@@ -5,10 +5,6 @@ ConditionPathIsMountPoint=!/media/configdrive
 # Only mount config drive block devices automatically in virtual machines
 ConditionVirtualization=vm
 
-# OpenStack defined config drive so they get to stick their name in it
-Wants=user-cloudinit@media-configdrive-openstack-latest-user_data.service
-Before=user-cloudinit@media-configdrive-openstack-latest-user_data.service
-
 [Service]
 Type=oneshot
 RemainAfterExit=no

units/configdrive-virtfs.service

@@ -4,10 +4,6 @@ Conflicts=configdrive-block.service umount.target
 ConditionPathIsMountPoint=!/media/configdrive
 ConditionVirtualization=vm
 
-# OpenStack defined config drive so they get to stick their name in it
-Wants=user-cloudinit@media-configdrive-openstack-latest-user_data.service
-Before=user-cloudinit@media-configdrive-openstack-latest-user_data.service
-
 # Support old style setup for now
 Wants=addon-run@media-configdrive.service addon-config@media-configdrive.service
 Before=addon-run@media-configdrive.service addon-config@media-configdrive.service

units/user-cloudinit@.path

@@ -0,0 +1,5 @@
+[Unit]
+Description=Watch for a cloud-config at %f
+
+[Path]
+PathExists=%f

units/user-config.target

@@ -3,9 +3,11 @@ Description=Load user-provided cloud configs
 Requires=system-config.target
 After=system-config.target
 
-# Load user_data placed by coreos-install
-Requires=user-cloudinit@var-lib-coreos\x2dinstall-user_data.service
-After=user-cloudinit@var-lib-coreos\x2dinstall-user_data.service
+# Watch for configs at a couple common paths
+Requires=user-cloudinit@media-configdrive-openstack-latest-user_data.path
+After=user-cloudinit@media-configdrive-openstack-latest-user_data.path
+Requires=user-cloudinit@var-lib-coreos\x2dinstall-user_data.path
+After=user-cloudinit@var-lib-coreos\x2dinstall-user_data.path
 
 Requires=user-cloudinit-proc-cmdline.service
 After=user-cloudinit-proc-cmdline.service