From 112ba1e31f908a426764f5a854dbc3cc79585f6e Mon Sep 17 00:00:00 2001 From: cnelson Date: Tue, 9 Dec 2014 08:10:23 -0800 Subject: [PATCH] Added support for the encoding key in write_files Supported encodings are base64, gzip, and base64 encoded gzip --- Documentation/cloud-config.md | 24 +++++++++- config/file.go | 2 +- system/file.go | 60 +++++++++++++++++++++-- system/file_test.go | 89 ++++++++++++++++++++++++++++++++++- 4 files changed, 168 insertions(+), 7 deletions(-) diff --git a/Documentation/cloud-config.md b/Documentation/cloud-config.md index 0895358..79f93f9 100644 --- a/Documentation/cloud-config.md +++ b/Documentation/cloud-config.md @@ -374,9 +374,11 @@ Each item in the list may have the following keys: - **content**: Data to write at the provided `path` - **permissions**: Integer representing file permissions, typically in octal notation (i.e. 0644) - **owner**: User and group that should own the file written to disk. This is equivalent to the `:` argument to `chown : `. +- **encoding**: Optional. The encoding of the data in content. If not specified this defaults to the yaml document encoding (usually utf-8). Supported encoding types are: + - **b64, base64**: Base64 encoded content + - **gz, gzip**: gzip encoded content, for use with the !!binary tag + - **gz+b64, gz+base64, gzip+b64, gzip+base64**: Base64 encoded gzip content -Explicitly not implemented is the **encoding** attribute. -The **content** field must represent exactly what should be written to disk. ```yaml #cloud-config @@ -391,6 +393,24 @@ write_files: owner: root content: | Good news, everyone! + - path: /tmp/like_this + permissions: 0644 + owner: root + encoding: gzip + content: !!binary | + H4sIAKgdh1QAAwtITM5WyK1USMqvUCjPLMlQSMssS1VIya9KzVPIySwszS9SyCpNLwYARQFQ5CcAAAA= + - path: /tmp/or_like_this + permissions: 0644 + owner: root + encoding: gzip+base64 + content: | + H4sIAKgdh1QAAwtITM5WyK1USMqvUCjPLMlQSMssS1VIya9KzVPIySwszS9SyCpNLwYARQFQ5CcAAAA= + - path: /tmp/todolist + permissions: 0644 + owner: root + encoding: base64 + content: | + UGFjayBteSBib3ggd2l0aCBmaXZlIGRvemVuIGxpcXVvciBqdWdz ``` ### manage_etc_hosts diff --git a/config/file.go b/config/file.go index 3aaa665..26dbc99 100644 --- a/config/file.go +++ b/config/file.go @@ -17,7 +17,7 @@ package config type File struct { - Encoding string `yaml:"-"` + Encoding string `yaml:"encoding" valid:"base64,b64,gz,gzip,gz+base64,gzip+base64,gz+b64,gzip+b64"` Content string `yaml:"content"` Owner string `yaml:"owner"` Path string `yaml:"path"` diff --git a/system/file.go b/system/file.go index ad8673f..312caf4 100644 --- a/system/file.go +++ b/system/file.go @@ -17,6 +17,9 @@ package system import ( + "bytes" + "compress/gzip" + "encoding/base64" "fmt" "io/ioutil" "log" @@ -47,13 +50,64 @@ func (f *File) Permissions() (os.FileMode, error) { return os.FileMode(perm), nil } +func DecodeBase64Content(content string) ([]byte, error) { + output, err := base64.StdEncoding.DecodeString(content) + + if err != nil { + return nil, fmt.Errorf("Unable to decode base64: %v", err) + } + + return output, nil +} + +func DecodeGzipContent(content string) ([]byte, error) { + gzr, err := gzip.NewReader(bytes.NewReader([]byte(content))) + + if err != nil { + return nil, fmt.Errorf("Unable to decode gzip: %v", err) + } + defer gzr.Close() + + buf := new(bytes.Buffer) + buf.ReadFrom(gzr) + + return buf.Bytes(), nil +} + +func DecodeContent(content string, encoding string) ([]byte, error) { + switch encoding { + case "": + return []byte(content), nil + + case "b64", "base64": + return DecodeBase64Content(content) + + case "gz", "gzip": + return DecodeGzipContent(content) + + case "gz+base64", "gzip+base64", "gz+b64", "gzip+b64": + gz, err := DecodeBase64Content(content) + + if err != nil { + return nil, err + } + + return DecodeGzipContent(string(gz)) + } + + return nil, fmt.Errorf("Unsupported encoding %s", encoding) + +} + func WriteFile(f *File, root string) (string, error) { fullpath := path.Join(root, f.Path) dir := path.Dir(fullpath) log.Printf("Writing file to %q", fullpath) - if f.Encoding != "" { - return "", fmt.Errorf("Unable to write file with encoding %s", f.Encoding) + content, err := DecodeContent(f.Content, f.Encoding) + + if err != nil { + return "", fmt.Errorf("Unable to decode %s (%v)", f.Path, err) } if err := EnsureDirectoryExists(dir); err != nil { @@ -71,7 +125,7 @@ func WriteFile(f *File, root string) (string, error) { return "", err } - if err := ioutil.WriteFile(tmp.Name(), []byte(f.Content), perm); err != nil { + if err := ioutil.WriteFile(tmp.Name(), content, perm); err != nil { return "", err } diff --git a/system/file_test.go b/system/file_test.go index f520649..8bd441b 100644 --- a/system/file_test.go +++ b/system/file_test.go @@ -156,10 +156,97 @@ func TestWriteFileEncodedContent(t *testing.T) { } defer os.RemoveAll(dir) + //all of these decode to "bar" + content_tests := map[string]string{ + "base64": "YmFy", + "b64": "YmFy", + "gz": "\x1f\x8b\x08\x08w\x14\x87T\x02\xffok\x00KJ,\x02\x00\xaa\x8c\xffv\x03\x00\x00\x00", + "gzip": "\x1f\x8b\x08\x08w\x14\x87T\x02\xffok\x00KJ,\x02\x00\xaa\x8c\xffv\x03\x00\x00\x00", + "gz+base64": "H4sIABMVh1QAA0tKLAIAqoz/dgMAAAA=", + "gzip+base64": "H4sIABMVh1QAA0tKLAIAqoz/dgMAAAA=", + "gz+b64": "H4sIABMVh1QAA0tKLAIAqoz/dgMAAAA=", + "gzip+b64": "H4sIABMVh1QAA0tKLAIAqoz/dgMAAAA=", + } + + for encoding, content := range content_tests { + fullPath := path.Join(dir, encoding) + + wf := File{config.File{ + Path: encoding, + Encoding: encoding, + Content: content, + RawFilePermissions: "0644", + }} + + path, err := WriteFile(&wf, dir) + if err != nil { + t.Fatalf("Processing of WriteFile failed: %v", err) + } else if path != fullPath { + t.Fatalf("WriteFile returned bad path: want %s, got %s", fullPath, path) + } + + fi, err := os.Stat(fullPath) + if err != nil { + t.Fatalf("Unable to stat file: %v", err) + } + + if fi.Mode() != os.FileMode(0644) { + t.Errorf("File has incorrect mode: %v", fi.Mode()) + } + + contents, err := ioutil.ReadFile(fullPath) + if err != nil { + t.Fatalf("Unable to read expected file: %v", err) + } + + if string(contents) != "bar" { + t.Fatalf("File has incorrect contents: '%s'", contents) + } + } +} + +func TestWriteFileInvalidEncodedContent(t *testing.T) { + dir, err := ioutil.TempDir(os.TempDir(), "coreos-cloudinit-") + if err != nil { + t.Fatalf("Unable to create tempdir: %v", err) + } + defer os.RemoveAll(dir) + + content_encodings := []string{ + "base64", + "b64", + "gz", + "gzip", + "gz+base64", + "gzip+base64", + "gz+b64", + "gzip+b64", + } + + for _, encoding := range content_encodings { + wf := File{config.File{ + Path: path.Join(dir, "tmp", "foo"), + Content: "@&*#%invalid data*@&^#*&", + Encoding: encoding, + }} + + if _, err := WriteFile(&wf, dir); err == nil { + t.Fatalf("Expected error to be raised when writing file with encoding") + } + } +} + +func TestWriteFileUnknownEncodedContent(t *testing.T) { + dir, err := ioutil.TempDir(os.TempDir(), "coreos-cloudinit-") + if err != nil { + t.Fatalf("Unable to create tempdir: %v", err) + } + defer os.RemoveAll(dir) + wf := File{config.File{ Path: path.Join(dir, "tmp", "foo"), Content: "", - Encoding: "base64", + Encoding: "no-such-encoding", }} if _, err := WriteFile(&wf, dir); err == nil {