Added support for the encoding key in write_files

Supported encodings are base64, gzip, and base64 encoded gzip

Documentation/cloud-config.md

@@ -374,9 +374,11 @@ Each item in the list may have the following keys:
 - **content**: Data to write at the provided `path`
 - **permissions**: Integer representing file permissions, typically in octal notation (i.e. 0644)
 - **owner**: User and group that should own the file written to disk. This is equivalent to the `<user>:<group>` argument to `chown <user>:<group> <path>`.
+- **encoding**: Optional. The encoding of the data in content. If not specified this defaults to the yaml document encoding (usually utf-8). Supported encoding types are:
+    - **b64, base64**: Base64 encoded content
+    - **gz, gzip**: gzip encoded content, for use with the !!binary tag
+    - **gz+b64, gz+base64, gzip+b64, gzip+base64**: Base64 encoded gzip content
 
-Explicitly not implemented is the **encoding** attribute.
-The **content** field must represent exactly what should be written to disk.
 
 ```yaml
 #cloud-config
@@ -391,6 +393,24 @@ write_files:
     owner: root
     content: |
       Good news, everyone!
+  - path: /tmp/like_this
+    permissions: 0644
+    owner: root
+    encoding: gzip
+    content: !!binary |
+      H4sIAKgdh1QAAwtITM5WyK1USMqvUCjPLMlQSMssS1VIya9KzVPIySwszS9SyCpNLwYARQFQ5CcAAAA=
+  - path: /tmp/or_like_this
+    permissions: 0644
+    owner: root
+    encoding: gzip+base64
+    content: |
+      H4sIAKgdh1QAAwtITM5WyK1USMqvUCjPLMlQSMssS1VIya9KzVPIySwszS9SyCpNLwYARQFQ5CcAAAA=
+  - path: /tmp/todolist
+    permissions: 0644
+    owner: root
+    encoding: base64
+    content: |
+      UGFjayBteSBib3ggd2l0aCBmaXZlIGRvemVuIGxpcXVvciBqdWdz
 ```
 
 ### manage_etc_hosts

config/file.go

@@ -17,7 +17,7 @@
 package config
 
 type File struct {
-	Encoding           string `yaml:"-"`
+	Encoding           string `yaml:"encoding" valid:"base64,b64,gz,gzip,gz+base64,gzip+base64,gz+b64,gzip+b64"`
 	Content            string `yaml:"content"`
 	Owner              string `yaml:"owner"`
 	Path               string `yaml:"path"`

system/file.go

@@ -17,6 +17,9 @@
 package system
 
 import (
+	"bytes"
+	"compress/gzip"
+	"encoding/base64"
 	"fmt"
 	"io/ioutil"
 	"log"
@@ -47,13 +50,64 @@ func (f *File) Permissions() (os.FileMode, error) {
 	return os.FileMode(perm), nil
 }
 
+func DecodeBase64Content(content string) ([]byte, error) {
+	output, err := base64.StdEncoding.DecodeString(content)
+
+	if err != nil {
+		return nil, fmt.Errorf("Unable to decode base64: %v", err)
+	}
+
+	return output, nil
+}
+
+func DecodeGzipContent(content string) ([]byte, error) {
+	gzr, err := gzip.NewReader(bytes.NewReader([]byte(content)))
+
+	if err != nil {
+		return nil, fmt.Errorf("Unable to decode gzip: %v", err)
+	}
+	defer gzr.Close()
+
+	buf := new(bytes.Buffer)
+	buf.ReadFrom(gzr)
+
+	return buf.Bytes(), nil
+}
+
+func DecodeContent(content string, encoding string) ([]byte, error) {
+	switch encoding {
+	case "":
+		return []byte(content), nil
+
+	case "b64", "base64":
+		return DecodeBase64Content(content)
+
+	case "gz", "gzip":
+		return DecodeGzipContent(content)
+
+	case "gz+base64", "gzip+base64", "gz+b64", "gzip+b64":
+		gz, err := DecodeBase64Content(content)
+
+		if err != nil {
+			return nil, err
+		}
+
+		return DecodeGzipContent(string(gz))
+	}
+
+	return nil, fmt.Errorf("Unsupported encoding %s", encoding)
+
+}
+
 func WriteFile(f *File, root string) (string, error) {
 	fullpath := path.Join(root, f.Path)
 	dir := path.Dir(fullpath)
 	log.Printf("Writing file to %q", fullpath)
 
-	if f.Encoding != "" {
-		return "", fmt.Errorf("Unable to write file with encoding %s", f.Encoding)
+	content, err := DecodeContent(f.Content, f.Encoding)
+
+	if err != nil {
+		return "", fmt.Errorf("Unable to decode %s (%v)", f.Path, err)
 	}
 
 	if err := EnsureDirectoryExists(dir); err != nil {
@@ -71,7 +125,7 @@ func WriteFile(f *File, root string) (string, error) {
 		return "", err
 	}
 
-	if err := ioutil.WriteFile(tmp.Name(), []byte(f.Content), perm); err != nil {
+	if err := ioutil.WriteFile(tmp.Name(), content, perm); err != nil {
 		return "", err
 	}
 

system/file_test.go

@@ -156,10 +156,97 @@ func TestWriteFileEncodedContent(t *testing.T) {
 	}
 	defer os.RemoveAll(dir)
 
+	//all of these decode to "bar"
+	content_tests := map[string]string{
+		"base64":      "YmFy",
+		"b64":         "YmFy",
+		"gz":          "\x1f\x8b\x08\x08w\x14\x87T\x02\xffok\x00KJ,\x02\x00\xaa\x8c\xffv\x03\x00\x00\x00",
+		"gzip":        "\x1f\x8b\x08\x08w\x14\x87T\x02\xffok\x00KJ,\x02\x00\xaa\x8c\xffv\x03\x00\x00\x00",
+		"gz+base64":   "H4sIABMVh1QAA0tKLAIAqoz/dgMAAAA=",
+		"gzip+base64": "H4sIABMVh1QAA0tKLAIAqoz/dgMAAAA=",
+		"gz+b64":      "H4sIABMVh1QAA0tKLAIAqoz/dgMAAAA=",
+		"gzip+b64":    "H4sIABMVh1QAA0tKLAIAqoz/dgMAAAA=",
+	}
+
+	for encoding, content := range content_tests {
+		fullPath := path.Join(dir, encoding)
+
+		wf := File{config.File{
+			Path:               encoding,
+			Encoding:           encoding,
+			Content:            content,
+			RawFilePermissions: "0644",
+		}}
+
+		path, err := WriteFile(&wf, dir)
+		if err != nil {
+			t.Fatalf("Processing of WriteFile failed: %v", err)
+		} else if path != fullPath {
+			t.Fatalf("WriteFile returned bad path: want %s, got %s", fullPath, path)
+		}
+
+		fi, err := os.Stat(fullPath)
+		if err != nil {
+			t.Fatalf("Unable to stat file: %v", err)
+		}
+
+		if fi.Mode() != os.FileMode(0644) {
+			t.Errorf("File has incorrect mode: %v", fi.Mode())
+		}
+
+		contents, err := ioutil.ReadFile(fullPath)
+		if err != nil {
+			t.Fatalf("Unable to read expected file: %v", err)
+		}
+
+		if string(contents) != "bar" {
+			t.Fatalf("File has incorrect contents: '%s'", contents)
+		}
+	}
+}
+
+func TestWriteFileInvalidEncodedContent(t *testing.T) {
+	dir, err := ioutil.TempDir(os.TempDir(), "coreos-cloudinit-")
+	if err != nil {
+		t.Fatalf("Unable to create tempdir: %v", err)
+	}
+	defer os.RemoveAll(dir)
+
+	content_encodings := []string{
+		"base64",
+		"b64",
+		"gz",
+		"gzip",
+		"gz+base64",
+		"gzip+base64",
+		"gz+b64",
+		"gzip+b64",
+	}
+
+	for _, encoding := range content_encodings {
+		wf := File{config.File{
+			Path:     path.Join(dir, "tmp", "foo"),
+			Content:  "@&*#%invalid data*@&^#*&",
+			Encoding: encoding,
+		}}
+
+		if _, err := WriteFile(&wf, dir); err == nil {
+			t.Fatalf("Expected error to be raised when writing file with encoding")
+		}
+	}
+}
+
+func TestWriteFileUnknownEncodedContent(t *testing.T) {
+	dir, err := ioutil.TempDir(os.TempDir(), "coreos-cloudinit-")
+	if err != nil {
+		t.Fatalf("Unable to create tempdir: %v", err)
+	}
+	defer os.RemoveAll(dir)
+
 	wf := File{config.File{
 		Path:     path.Join(dir, "tmp", "foo"),
 		Content:  "",
-		Encoding: "base64",
+		Encoding: "no-such-encoding",
 	}}
 
 	if _, err := WriteFile(&wf, dir); err == nil {