unistack-org/cloudinit
4
0
Fork 0
Code Issues 1 Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #279 from cnelson/write_files_encoding_support

Browse Source 
Add support for the encoding key to write_files
This commit is contained in:
Jonathan Boulle 2014-12-09 17:37:34 -08:00
commit 353444e56d
15 changed files with 527 additions and 161 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

24
Documentation/cloud-config.md
View File

@ -374,9 +374,11 @@ Each item in the list may have the following keys:
- **content**: Data to write at the provided `path`
- **permissions**: Integer representing file permissions, typically in octal notation (i.e. 0644)
- **owner**: User and group that should own the file written to disk. This is equivalent to the `<user>:<group>` argument to `chown <user>:<group> <path>`.
- **encoding**: Optional. The encoding of the data in content. If not specified this defaults to the yaml document encoding (usually utf-8). Supported encoding types are:
- **b64, base64**: Base64 encoded content
- **gz, gzip**: gzip encoded content, for use with the !!binary tag
- **gz+b64, gz+base64, gzip+b64, gzip+base64**: Base64 encoded gzip content
Explicitly not implemented is the **encoding** attribute.
The **content** field must represent exactly what should be written to disk.
```yaml
#cloud-config
@ -391,6 +393,24 @@ write_files:
owner: root
content: |
Good news, everyone!
- path: /tmp/like_this
permissions: 0644
owner: root
encoding: gzip
content: !!binary |
H4sIAKgdh1QAAwtITM5WyK1USMqvUCjPLMlQSMssS1VIya9KzVPIySwszS9SyCpNLwYARQFQ5CcAAAA=
- path: /tmp/or_like_this
permissions: 0644
owner: root
encoding: gzip+base64
content: |
H4sIAKgdh1QAAwtITM5WyK1USMqvUCjPLMlQSMssS1VIya9KzVPIySwszS9SyCpNLwYARQFQ5CcAAAA=
- path: /tmp/todolist
permissions: 0644
owner: root
encoding: base64
content: |
UGFjayBteSBib3ggd2l0aCBmaXZlIGRvemVuIGxpcXVvciBqdWdz
```
### manage_etc_hosts

2
Godeps/Godeps.json generated
View File

@ -28,7 +28,7 @@
},
{
"ImportPath": "gopkg.in/yaml.v1",
"Rev": "feb4ca79644e8e7e39c06095246ee54b1282c118"
"Rev": "9f9df34309c04878acc86042b16630b0f696e1de"
}
]
}

3
Godeps/_workspace/src/gopkg.in/yaml.v1/LICENSE generated vendored
View File

@ -1,3 +1,6 @@
Copyright (c) 2011-2014 - Canonical Inc.
This software is licensed under the LGPLv3, included below.
As a special exception to the GNU Lesser General Public License version 3

8
Godeps/_workspace/src/gopkg.in/yaml.v1/README.md generated vendored
View File

@ -12,10 +12,10 @@ C library to parse and generate YAML data quickly and reliably.
Compatibility
-------------
The yaml package is almost compatible with YAML 1.1, including support for
anchors, tags, etc. There are still a few missing bits, such as document
merging, base-60 floats (huh?), and multi-document unmarshalling. These
features are not hard to add, and will be introduced as necessary.
The yaml package supports most of YAML 1.1 and 1.2, including support for
anchors, tags, map merging, etc. Multi-document unmarshalling is not yet
implemented, and base-60 floats from YAML 1.1 are purposefully not
supported since they're a poor design and are gone in YAML 1.2.
Installation and usage
----------------------

92
Godeps/_workspace/src/gopkg.in/yaml.v1/decode.go generated vendored
View File

@ -1,6 +1,8 @@
package yaml
import (
"encoding/base64"
"fmt"
"reflect"
"strconv"
"time"
@ -63,7 +65,7 @@ func (p *parser) destroy() {
func (p *parser) skip() {
if p.event.typ != yaml_NO_EVENT {
if p.event.typ == yaml_STREAM_END_EVENT {
panic("Attempted to go past the end of stream. Corrupted value?")
fail("Attempted to go past the end of stream. Corrupted value?")
}
yaml_event_delete(&p.event)
}
@ -89,7 +91,7 @@ func (p *parser) fail() {
} else {
msg = "Unknown problem parsing YAML content"
}
panic(where + msg)
fail(where + msg)
}
func (p *parser) anchor(n *node, anchor []byte) {
@ -114,10 +116,9 @@ func (p *parser) parse() *node {
// Happens when attempting to decode an empty buffer.
return nil
default:
panic("Attempted to parse unknown event: " +
strconv.Itoa(int(p.event.typ)))
panic("Attempted to parse unknown event: " + strconv.Itoa(int(p.event.typ)))
}
panic("Unreachable")
panic("unreachable")
}
func (p *parser) node(kind int) *node {
@ -135,8 +136,7 @@ func (p *parser) document() *node {
p.skip()
n.children = append(n.children, p.parse())
if p.event.typ != yaml_DOCUMENT_END_EVENT {
panic("Expected end of document event but got " +
strconv.Itoa(int(p.event.typ)))
panic("Expected end of document event but got " + strconv.Itoa(int(p.event.typ)))
}
p.skip()
return n
@ -218,7 +218,7 @@ func (d *decoder) setter(tag string, out *reflect.Value, good *bool) (set func()
var arg interface{}
*out = reflect.ValueOf(&arg).Elem()
return func() {
*good = setter.SetYAML(tag, arg)
*good = setter.SetYAML(shortTag(tag), arg)
}
}
}
@ -226,7 +226,7 @@ func (d *decoder) setter(tag string, out *reflect.Value, good *bool) (set func()
for again {
again = false
setter, _ := (*out).Interface().(Setter)
if tag != "!!null" || setter != nil {
if tag != yaml_NULL_TAG || setter != nil {
if pv := (*out); pv.Kind() == reflect.Ptr {
if pv.IsNil() {
*out = reflect.New(pv.Type().Elem()).Elem()
@ -242,7 +242,7 @@ func (d *decoder) setter(tag string, out *reflect.Value, good *bool) (set func()
var arg interface{}
*out = reflect.ValueOf(&arg).Elem()
return func() {
*good = setter.SetYAML(tag, arg)
*good = setter.SetYAML(shortTag(tag), arg)
}
}
}
@ -279,10 +279,10 @@ func (d *decoder) document(n *node, out reflect.Value) (good bool) {
func (d *decoder) alias(n *node, out reflect.Value) (good bool) {
an, ok := d.doc.anchors[n.value]
if !ok {
panic("Unknown anchor '" + n.value + "' referenced")
fail("Unknown anchor '" + n.value + "' referenced")
}
if d.aliases[n.value] {
panic("Anchor '" + n.value + "' value contains itself")
fail("Anchor '" + n.value + "' value contains itself")
}
d.aliases[n.value] = true
good = d.unmarshal(an, out)
@ -290,23 +290,50 @@ func (d *decoder) alias(n *node, out reflect.Value) (good bool) {
return good
}
var zeroValue reflect.Value
func resetMap(out reflect.Value) {
for _, k := range out.MapKeys() {
out.SetMapIndex(k, zeroValue)
}
}
var durationType = reflect.TypeOf(time.Duration(0))
func (d *decoder) scalar(n *node, out reflect.Value) (good bool) {
var tag string
var resolved interface{}
if n.tag == "" && !n.implicit {
tag = "!!str"
tag = yaml_STR_TAG
resolved = n.value
} else {
tag, resolved = resolve(n.tag, n.value)
if tag == yaml_BINARY_TAG {
data, err := base64.StdEncoding.DecodeString(resolved.(string))
if err != nil {
fail("!!binary value contains invalid base64 data")
}
resolved = string(data)
}
}
if set := d.setter(tag, &out, &good); set != nil {
defer set()
}
if resolved == nil {
if out.Kind() == reflect.Map && !out.CanAddr() {
resetMap(out)
} else {
out.Set(reflect.Zero(out.Type()))
}
good = true
return
}
switch out.Kind() {
case reflect.String:
if resolved != nil {
if tag == yaml_BINARY_TAG {
out.SetString(resolved.(string))
good = true
} else if resolved != nil {
out.SetString(n.value)
good = true
}
@ -380,17 +407,11 @@ func (d *decoder) scalar(n *node, out reflect.Value) (good bool) {
good = true
}
case reflect.Ptr:
switch resolved.(type) {
case nil:
out.Set(reflect.Zero(out.Type()))
if out.Type().Elem() == reflect.TypeOf(resolved) {
elem := reflect.New(out.Type().Elem())
elem.Elem().Set(reflect.ValueOf(resolved))
out.Set(elem)
good = true
default:
if out.Type().Elem() == reflect.TypeOf(resolved) {
elem := reflect.New(out.Type().Elem())
elem.Elem().Set(reflect.ValueOf(resolved))
out.Set(elem)
good = true
}
}
}
return good
@ -404,7 +425,7 @@ func settableValueOf(i interface{}) reflect.Value {
}
func (d *decoder) sequence(n *node, out reflect.Value) (good bool) {
if set := d.setter("!!seq", &out, &good); set != nil {
if set := d.setter(yaml_SEQ_TAG, &out, &good); set != nil {
defer set()
}
var iface reflect.Value
@ -433,7 +454,7 @@ func (d *decoder) sequence(n *node, out reflect.Value) (good bool) {
}
func (d *decoder) mapping(n *node, out reflect.Value) (good bool) {
if set := d.setter("!!map", &out, &good); set != nil {
if set := d.setter(yaml_MAP_TAG, &out, &good); set != nil {
defer set()
}
if out.Kind() == reflect.Struct {
@ -465,6 +486,13 @@ func (d *decoder) mapping(n *node, out reflect.Value) (good bool) {
}
k := reflect.New(kt).Elem()
if d.unmarshal(n.children[i], k) {
kkind := k.Kind()
if kkind == reflect.Interface {
kkind = k.Elem().Kind()
}
if kkind == reflect.Map || kkind == reflect.Slice {
fail(fmt.Sprintf("invalid map key: %#v", k.Interface()))
}
e := reflect.New(et).Elem()
if d.unmarshal(n.children[i+1], e) {
out.SetMapIndex(k, e)
@ -511,28 +539,28 @@ func (d *decoder) merge(n *node, out reflect.Value) {
case aliasNode:
an, ok := d.doc.anchors[n.value]
if ok && an.kind != mappingNode {
panic(wantMap)
fail(wantMap)
}
d.unmarshal(n, out)
case sequenceNode:
// Step backwards as earlier nodes take precedence.
for i := len(n.children)-1; i >= 0; i-- {
for i := len(n.children) - 1; i >= 0; i-- {
ni := n.children[i]
if ni.kind == aliasNode {
an, ok := d.doc.anchors[ni.value]
if ok && an.kind != mappingNode {
panic(wantMap)
fail(wantMap)
}
} else if ni.kind != mappingNode {
panic(wantMap)
fail(wantMap)
}
d.unmarshal(ni, out)
}
default:
panic(wantMap)
fail(wantMap)
}
}
func isMerge(n *node) bool {
return n.kind == scalarNode && n.value == "<<" && (n.implicit == true || n.tag == "!!merge" || n.tag == "tag:yaml.org,2002:merge")
return n.kind == scalarNode && n.value == "<<" && (n.implicit == true || n.tag == yaml_MERGE_TAG)
}

55
Godeps/_workspace/src/gopkg.in/yaml.v1/decode_test.go generated vendored
View File

@ -2,9 +2,10 @@ package yaml_test
import (
. "gopkg.in/check.v1"
"github.com/coreos/coreos-cloudinit/Godeps/_workspace/src/gopkg.in/yaml.v1"
"gopkg.in/yaml.v1"
"math"
"reflect"
"strings"
"time"
)
@ -316,7 +317,10 @@ var unmarshalTests = []struct {
map[string]*string{"foo": new(string)},
}, {
"foo: null",
map[string]string{},
map[string]string{"foo": ""},
}, {
"foo: null",
map[string]interface{}{"foo": nil},
},
// Ignored field
@ -377,6 +381,24 @@ var unmarshalTests = []struct {
"a: <foo>",
map[string]string{"a": "<foo>"},
},
// Base 60 floats are obsolete and unsupported.
{
"a: 1:1\n",
map[string]string{"a": "1:1"},
},
// Binary data.
{
"a: !!binary gIGC\n",
map[string]string{"a": "\x80\x81\x82"},
}, {
"a: !!binary |\n " + strings.Repeat("kJCQ", 17) + "kJ\n CQ\n",
map[string]string{"a": strings.Repeat("\x90", 54)},
}, {
"a: !!binary |\n " + strings.Repeat("A", 70) + "\n ==\n",
map[string]string{"a": strings.Repeat("\x00", 52)},
},
}
type inlineB struct {
@ -424,12 +446,15 @@ func (s *S) TestUnmarshalNaN(c *C) {
var unmarshalErrorTests = []struct {
data, error string
}{
{"v: !!float 'error'", "YAML error: Can't decode !!str 'error' as a !!float"},
{"v: !!float 'error'", "YAML error: cannot decode !!str `error` as a !!float"},
{"v: [A,", "YAML error: line 1: did not find expected node content"},
{"v:\n- [A,", "YAML error: line 2: did not find expected node content"},
{"a: *b\n", "YAML error: Unknown anchor 'b' referenced"},
{"a: &a\n b: *a\n", "YAML error: Anchor 'a' value contains itself"},
{"value: -", "YAML error: block sequence entries are not allowed in this context"},
{"a: !!binary ==", "YAML error: !!binary value contains invalid base64 data"},
{"{[.]}", `YAML error: invalid map key: \[\]interface \{\}\{"\."\}`},
{"{{.}}", `YAML error: invalid map key: map\[interface\ \{\}\]interface \{\}\{".":interface \{\}\(nil\)\}`},
}
func (s *S) TestUnmarshalErrors(c *C) {
@ -624,6 +649,30 @@ func (s *S) TestMergeStruct(c *C) {
}
}
var unmarshalNullTests = []func() interface{}{
func() interface{} { var v interface{}; v = "v"; return &v },
func() interface{} { var s = "s"; return &s },
func() interface{} { var s = "s"; sptr := &s; return &sptr },
func() interface{} { var i = 1; return &i },
func() interface{} { var i = 1; iptr := &i; return &iptr },
func() interface{} { m := map[string]int{"s": 1}; return &m },
func() interface{} { m := map[string]int{"s": 1}; return m },
}
func (s *S) TestUnmarshalNull(c *C) {
for _, test := range unmarshalNullTests {
item := test()
zero := reflect.Zero(reflect.TypeOf(item).Elem()).Interface()
err := yaml.Unmarshal([]byte("null"), item)
c.Assert(err, IsNil)
if reflect.TypeOf(item).Kind() == reflect.Map {
c.Assert(reflect.ValueOf(item).Interface(), DeepEquals, reflect.MakeMap(reflect.TypeOf(item)).Interface())
} else {
c.Assert(reflect.ValueOf(item).Elem().Interface(), DeepEquals, zero)
}
}
}
//var data []byte
//func init() {
// var err error

5
Godeps/_workspace/src/gopkg.in/yaml.v1/emitterc.go generated vendored
View File

@ -973,8 +973,8 @@ func yaml_emitter_analyze_tag(emitter *yaml_emitter_t, tag []byte) bool {
if bytes.HasPrefix(tag, tag_directive.prefix) {
emitter.tag_data.handle = tag_directive.handle
emitter.tag_data.suffix = tag[len(tag_directive.prefix):]
return true
}
return true
}
emitter.tag_data.suffix = tag
return true
@ -1279,6 +1279,9 @@ func yaml_emitter_write_tag_content(emitter *yaml_emitter_t, value []byte, need_
for k := 0; k < w; k++ {
octet := value[i]
i++
if !put(emitter, '%') {
return false
}
c := octet >> 4
if c < 10 {

51
Godeps/_workspace/src/gopkg.in/yaml.v1/encode.go generated vendored
View File

@ -2,8 +2,10 @@ package yaml
import (
"reflect"
"regexp"
"sort"
"strconv"
"strings"
"time"
)
@ -50,14 +52,19 @@ func (e *encoder) must(ok bool) {
if msg == "" {
msg = "Unknown problem generating YAML content"
}
panic(msg)
fail(msg)
}
}
func (e *encoder) marshal(tag string, in reflect.Value) {
if !in.IsValid() {
e.nilv()
return
}
var value interface{}
if getter, ok := in.Interface().(Getter); ok {
tag, value = getter.GetYAML()
tag = longTag(tag)
if value == nil {
e.nilv()
return
@ -98,7 +105,7 @@ func (e *encoder) marshal(tag string, in reflect.Value) {
case reflect.Bool:
e.boolv(tag, in)
default:
panic("Can't marshal type yet: " + in.Type().String())
panic("Can't marshal type: " + in.Type().String())
}
}
@ -167,11 +174,46 @@ func (e *encoder) slicev(tag string, in reflect.Value) {
e.emit()
}
// isBase60 returns whether s is in base 60 notation as defined in YAML 1.1.
//
// The base 60 float notation in YAML 1.1 is a terrible idea and is unsupported
// in YAML 1.2 and by this package, but these should be marshalled quoted for
// the time being for compatibility with other parsers.
func isBase60Float(s string) (result bool) {
// Fast path.
if s == "" {
return false
}
c := s[0]
if !(c == '+' || c == '-' || c >= '0' && c <= '9') || strings.IndexByte(s, ':') < 0 {
return false
}
// Do the full match.
return base60float.MatchString(s)
}
// From http://yaml.org/type/float.html, except the regular expression there
// is bogus. In practice parsers do not enforce the "\.[0-9_]*" suffix.
var base60float = regexp.MustCompile(`^[-+]?[0-9][0-9_]*(?::[0-5]?[0-9])+(?:\.[0-9_]*)?$`)
func (e *encoder) stringv(tag string, in reflect.Value) {
var style yaml_scalar_style_t
s := in.String()
if rtag, _ := resolve("", s); rtag != "!!str" {
rtag, rs := resolve("", s)
if rtag == yaml_BINARY_TAG {
if tag == "" || tag == yaml_STR_TAG {
tag = rtag
s = rs.(string)
} else if tag == yaml_BINARY_TAG {
fail("explicitly tagged !!binary data must be base64-encoded")
} else {
fail("cannot marshal invalid UTF-8 data as " + shortTag(tag))
}
}
if tag == "" && (rtag != yaml_STR_TAG || isBase60Float(s)) {
style = yaml_DOUBLE_QUOTED_SCALAR_STYLE
} else if strings.Contains(s, "\n") {
style = yaml_LITERAL_SCALAR_STYLE
} else {
style = yaml_PLAIN_SCALAR_STYLE
}
@ -218,9 +260,6 @@ func (e *encoder) nilv() {
func (e *encoder) emitScalar(value, anchor, tag string, style yaml_scalar_style_t) {
implicit := tag == ""
if !implicit {
style = yaml_PLAIN_SCALAR_STYLE
}
e.must(yaml_scalar_event_initialize(&e.event, []byte(anchor), []byte(tag), []byte(value), implicit, implicit, style))
e.emit()
}

71
Godeps/_workspace/src/gopkg.in/yaml.v1/encode_test.go generated vendored
View File

@ -2,12 +2,13 @@ package yaml_test
import (
"fmt"
"github.com/coreos/coreos-cloudinit/Godeps/_workspace/src/gopkg.in/yaml.v1"
. "gopkg.in/check.v1"
"math"
"strconv"
"strings"
"time"
. "gopkg.in/check.v1"
"gopkg.in/yaml.v1"
)
var marshalIntTest = 123
@ -17,6 +18,9 @@ var marshalTests = []struct {
data string
}{
{
nil,
"null\n",
}, {
&struct{}{},
"{}\n",
}, {
@ -87,7 +91,7 @@ var marshalTests = []struct {
"v:\n- A\n- B\n",
}, {
map[string][]string{"v": []string{"A", "B\nC"}},
"v:\n- A\n- 'B\n\n C'\n",
"v:\n- A\n- |-\n B\n C\n",
}, {
map[string][]interface{}{"v": []interface{}{"A", 1, map[string][]int{"B": []int{2, 3}}}},
"v:\n- A\n- 1\n- B:\n - 2\n - 3\n",
@ -220,11 +224,39 @@ var marshalTests = []struct {
"a: 3s\n",
},
// Issue #24.
// Issue #24: bug in map merging logic.
{
map[string]string{"a": "<foo>"},
"a: <foo>\n",
},
// Issue #34: marshal unsupported base 60 floats quoted for compatibility
// with old YAML 1.1 parsers.
{
map[string]string{"a": "1:1"},
"a: \"1:1\"\n",
},
// Binary data.
{
map[string]string{"a": "\x00"},
"a: \"\\0\"\n",
}, {
map[string]string{"a": "\x80\x81\x82"},
"a: !!binary gIGC\n",
}, {
map[string]string{"a": strings.Repeat("\x90", 54)},
"a: !!binary |\n " + strings.Repeat("kJCQ", 17) + "kJ\n CQ\n",
}, {
map[string]interface{}{"a": typeWithGetter{"!!str", "\x80\x81\x82"}},
"a: !!binary gIGC\n",
},
// Escaping of tags.
{
map[string]interface{}{"a": typeWithGetter{"foo!bar", 1}},
"a: !<foo%21bar> 1\n",
},
}
func (s *S) TestMarshal(c *C) {
@ -238,20 +270,29 @@ func (s *S) TestMarshal(c *C) {
var marshalErrorTests = []struct {
value interface{}
error string
}{
{
&struct {
B int
inlineB ",inline"
}{1, inlineB{2, inlineC{3}}},
`Duplicated key 'b' in struct struct \{ B int; .*`,
},
}
panic string
}{{
value: &struct {
B int
inlineB ",inline"
}{1, inlineB{2, inlineC{3}}},
panic: `Duplicated key 'b' in struct struct \{ B int; .*`,
}, {
value: typeWithGetter{"!!binary", "\x80"},
error: "YAML error: explicitly tagged !!binary data must be base64-encoded",
}, {
value: typeWithGetter{"!!float", "\x80"},
error: `YAML error: cannot marshal invalid UTF-8 data as !!float`,
}}
func (s *S) TestMarshalErrors(c *C) {
for _, item := range marshalErrorTests {
_, err := yaml.Marshal(item.value)
c.Assert(err, ErrorMatches, item.error)
if item.panic != "" {
c.Assert(func() { yaml.Marshal(item.value) }, PanicMatches, item.panic)
} else {
_, err := yaml.Marshal(item.value)
c.Assert(err, ErrorMatches, item.error)
}
}
}

181
Godeps/_workspace/src/gopkg.in/yaml.v1/resolve.go generated vendored
View File

@ -1,9 +1,12 @@
package yaml
import (
"encoding/base64"
"fmt"
"math"
"strconv"
"strings"
"unicode/utf8"
)
// TODO: merge, timestamps, base 60 floats, omap.
@ -33,18 +36,18 @@ func init() {
tag string
l []string
}{
{true, "!!bool", []string{"y", "Y", "yes", "Yes", "YES"}},
{true, "!!bool", []string{"true", "True", "TRUE"}},
{true, "!!bool", []string{"on", "On", "ON"}},
{false, "!!bool", []string{"n", "N", "no", "No", "NO"}},
{false, "!!bool", []string{"false", "False", "FALSE"}},
{false, "!!bool", []string{"off", "Off", "OFF"}},
{nil, "!!null", []string{"~", "null", "Null", "NULL"}},
{math.NaN(), "!!float", []string{".nan", ".NaN", ".NAN"}},
{math.Inf(+1), "!!float", []string{".inf", ".Inf", ".INF"}},
{math.Inf(+1), "!!float", []string{"+.inf", "+.Inf", "+.INF"}},
{math.Inf(-1), "!!float", []string{"-.inf", "-.Inf", "-.INF"}},
{"<<", "!!merge", []string{"<<"}},
{true, yaml_BOOL_TAG, []string{"y", "Y", "yes", "Yes", "YES"}},
{true, yaml_BOOL_TAG, []string{"true", "True", "TRUE"}},
{true, yaml_BOOL_TAG, []string{"on", "On", "ON"}},
{false, yaml_BOOL_TAG, []string{"n", "N", "no", "No", "NO"}},
{false, yaml_BOOL_TAG, []string{"false", "False", "FALSE"}},
{false, yaml_BOOL_TAG, []string{"off", "Off", "OFF"}},
{nil, yaml_NULL_TAG, []string{"", "~", "null", "Null", "NULL"}},
{math.NaN(), yaml_FLOAT_TAG, []string{".nan", ".NaN", ".NAN"}},
{math.Inf(+1), yaml_FLOAT_TAG, []string{".inf", ".Inf", ".INF"}},
{math.Inf(+1), yaml_FLOAT_TAG, []string{"+.inf", "+.Inf", "+.INF"}},
{math.Inf(-1), yaml_FLOAT_TAG, []string{"-.inf", "-.Inf", "-.INF"}},
{"<<", yaml_MERGE_TAG, []string{"<<"}},
}
m := resolveMap
@ -58,90 +61,130 @@ func init() {
const longTagPrefix = "tag:yaml.org,2002:"
func shortTag(tag string) string {
// TODO This can easily be made faster and produce less garbage.
if strings.HasPrefix(tag, longTagPrefix) {
return "!!" + tag[len(longTagPrefix):]
}
return tag
}
func longTag(tag string) string {
if strings.HasPrefix(tag, "!!") {
return longTagPrefix + tag[2:]
}
return tag
}
func resolvableTag(tag string) bool {
switch tag {
case "", "!!str", "!!bool", "!!int", "!!float", "!!null":
case "", yaml_STR_TAG, yaml_BOOL_TAG, yaml_INT_TAG, yaml_FLOAT_TAG, yaml_NULL_TAG:
return true
}
return false
}
func resolve(tag string, in string) (rtag string, out interface{}) {
tag = shortTag(tag)
if !resolvableTag(tag) {
return tag, in
}
defer func() {
if tag != "" && tag != rtag {
panic("Can't decode " + rtag + " '" + in + "' as a " + tag)
switch tag {
case "", rtag, yaml_STR_TAG, yaml_BINARY_TAG:
return
}
fail(fmt.Sprintf("cannot decode %s `%s` as a %s", shortTag(rtag), in, shortTag(tag)))
}()
if in == "" {
return "!!null", nil
// Any data is accepted as a !!str or !!binary.
// Otherwise, the prefix is enough of a hint about what it might be.
hint := byte('N')
if in != "" {
hint = resolveTable[in[0]]
}
c := resolveTable[in[0]]
if c == 0 {
// It's a string for sure. Nothing to do.
return "!!str", in
}
// Handle things we can lookup in a map.
if item, ok := resolveMap[in]; ok {
return item.tag, item.value
}
switch c {
case 'M':
// We've already checked the map above.
case '.':
// Not in the map, so maybe a normal float.
floatv, err := strconv.ParseFloat(in, 64)
if err == nil {
return "!!float", floatv
if hint != 0 && tag != yaml_STR_TAG && tag != yaml_BINARY_TAG {
// Handle things we can lookup in a map.
if item, ok := resolveMap[in]; ok {
return item.tag, item.value
}
// XXX Handle base 60 floats here (WTF!)
case 'D', 'S':
// Int, float, or timestamp.
plain := strings.Replace(in, "_", "", -1)
intv, err := strconv.ParseInt(plain, 0, 64)
if err == nil {
if intv == int64(int(intv)) {
return "!!int", int(intv)
} else {
return "!!int", intv
}
}
floatv, err := strconv.ParseFloat(plain, 64)
if err == nil {
return "!!float", floatv
}
if strings.HasPrefix(plain, "0b") {
intv, err := strconv.ParseInt(plain[2:], 2, 64)
// Base 60 floats are a bad idea, were dropped in YAML 1.2, and
// are purposefully unsupported here. They're still quoted on
// the way out for compatibility with other parser, though.
switch hint {
case 'M':
// We've already checked the map above.
case '.':
// Not in the map, so maybe a normal float.
floatv, err := strconv.ParseFloat(in, 64)
if err == nil {
return "!!int", int(intv)
return yaml_FLOAT_TAG, floatv
}
} else if strings.HasPrefix(plain, "-0b") {
intv, err := strconv.ParseInt(plain[3:], 2, 64)
if err == nil {
return "!!int", -int(intv)
}
}
// XXX Handle timestamps here.
default:
panic("resolveTable item not yet handled: " +
string([]byte{c}) + " (with " + in + ")")
case 'D', 'S':
// Int, float, or timestamp.
plain := strings.Replace(in, "_", "", -1)
intv, err := strconv.ParseInt(plain, 0, 64)
if err == nil {
if intv == int64(int(intv)) {
return yaml_INT_TAG, int(intv)
} else {
return yaml_INT_TAG, intv
}
}
floatv, err := strconv.ParseFloat(plain, 64)
if err == nil {
return yaml_FLOAT_TAG, floatv
}
if strings.HasPrefix(plain, "0b") {
intv, err := strconv.ParseInt(plain[2:], 2, 64)
if err == nil {
return yaml_INT_TAG, int(intv)
}
} else if strings.HasPrefix(plain, "-0b") {
intv, err := strconv.ParseInt(plain[3:], 2, 64)
if err == nil {
return yaml_INT_TAG, -int(intv)
}
}
// XXX Handle timestamps here.
default:
panic("resolveTable item not yet handled: " + string(rune(hint)) + " (with " + in + ")")
}
}
return "!!str", in
if tag == yaml_BINARY_TAG {
return yaml_BINARY_TAG, in
}
if utf8.ValidString(in) {
return yaml_STR_TAG, in
}
return yaml_BINARY_TAG, encodeBase64(in)
}
// encodeBase64 encodes s as base64 that is broken up into multiple lines
// as appropriate for the resulting length.
func encodeBase64(s string) string {
const lineLen = 70
encLen := base64.StdEncoding.EncodedLen(len(s))
lines := encLen/lineLen + 1
buf := make([]byte, encLen*2+lines)
in := buf[0:encLen]
out := buf[encLen:]
base64.StdEncoding.Encode(in, []byte(s))
k := 0
for i := 0; i < len(in); i += lineLen {
j := i + lineLen
if j > len(in) {
j = len(in)
}
k += copy(out[k:], in[i:j])
if lines > 1 {
out[k] = '\n'
k++
}
}
return string(out[:k])
}

41
Godeps/_workspace/src/gopkg.in/yaml.v1/yaml.go generated vendored
View File

@ -10,23 +10,20 @@ import (
"errors"
"fmt"
"reflect"
"runtime"
"strings"
"sync"
)
type yamlError string
func fail(msg string) {
panic(yamlError(msg))
}
func handleErr(err *error) {
if r := recover(); r != nil {
if _, ok := r.(runtime.Error); ok {
panic(r)
} else if _, ok := r.(*reflect.ValueError); ok {
panic(r)
} else if _, ok := r.(externalPanic); ok {
panic(r)
} else if s, ok := r.(string); ok {
*err = errors.New("YAML error: " + s)
} else if e, ok := r.(error); ok {
*err = e
if e, ok := r.(yamlError); ok {
*err = errors.New("YAML error: " + string(e))
} else {
panic(r)
}
@ -78,7 +75,7 @@ type Getter interface {
// F int `yaml:"a,omitempty"`
// B int
// }
// var T t
// var t T
// yaml.Unmarshal([]byte("a: 1\nb: 2"), &t)
//
// See the documentation of Marshal for the format of tags and a list of
@ -91,7 +88,11 @@ func Unmarshal(in []byte, out interface{}) (err error) {
defer p.destroy()
node := p.parse()
if node != nil {
d.unmarshal(node, reflect.ValueOf(out))
v := reflect.ValueOf(out)
if v.Kind() == reflect.Ptr && !v.IsNil() {
v = v.Elem()
}
d.unmarshal(node, v)
}
return nil
}
@ -174,12 +175,6 @@ type fieldInfo struct {
var structMap = make(map[reflect.Type]*structInfo)
var fieldMapMutex sync.RWMutex
type externalPanic string
func (e externalPanic) String() string {
return string(e)
}
func getStructInfo(st reflect.Type) (*structInfo, error) {
fieldMapMutex.RLock()
sinfo, found := structMap[st]
@ -220,8 +215,7 @@ func getStructInfo(st reflect.Type) (*structInfo, error) {
case "inline":
inline = true
default:
msg := fmt.Sprintf("Unsupported flag %q in tag %q of type %s", flag, tag, st)
panic(externalPanic(msg))
return nil, errors.New(fmt.Sprintf("Unsupported flag %q in tag %q of type %s", flag, tag, st))
}
}
tag = fields[0]
@ -229,6 +223,7 @@ func getStructInfo(st reflect.Type) (*structInfo, error) {
if inline {
switch field.Type.Kind() {
// TODO: Implement support for inline maps.
//case reflect.Map:
// if inlineMap >= 0 {
// return nil, errors.New("Multiple ,inline maps in struct " + st.String())
@ -256,8 +251,8 @@ func getStructInfo(st reflect.Type) (*structInfo, error) {
fieldsList = append(fieldsList, finfo)
}
default:
//panic("Option ,inline needs a struct value or map field")
panic("Option ,inline needs a struct value field")
//return nil, errors.New("Option ,inline needs a struct value or map field")
return nil, errors.New("Option ,inline needs a struct value field")
}
continue
}

4
Godeps/_workspace/src/gopkg.in/yaml.v1/yamlh.go generated vendored
View File

@ -294,6 +294,10 @@ const (
yaml_SEQ_TAG = "tag:yaml.org,2002:seq" // The tag !!seq is used to denote sequences.
yaml_MAP_TAG = "tag:yaml.org,2002:map" // The tag !!map is used to denote mapping.
// Not in original libyaml.
yaml_BINARY_TAG = "tag:yaml.org,2002:binary"
yaml_MERGE_TAG = "tag:yaml.org,2002:merge"
yaml_DEFAULT_SCALAR_TAG = yaml_STR_TAG // The default scalar tag is !!str.
yaml_DEFAULT_SEQUENCE_TAG = yaml_SEQ_TAG // The default sequence tag is !!seq.
yaml_DEFAULT_MAPPING_TAG = yaml_MAP_TAG // The default mapping tag is !!map.

2
config/file.go
View File

@ -17,7 +17,7 @@
package config
type File struct {
Encoding string `yaml:"-"`
Encoding string `yaml:"encoding" valid:"base64,b64,gz,gzip,gz+base64,gzip+base64,gz+b64,gzip+b64"`
Content string `yaml:"content"`
Owner string `yaml:"owner"`
Path string `yaml:"path"`

60
system/file.go
View File

@ -17,6 +17,9 @@
package system
import (
"bytes"
"compress/gzip"
"encoding/base64"
"fmt"
"io/ioutil"
"log"
@ -47,13 +50,64 @@ func (f *File) Permissions() (os.FileMode, error) {
return os.FileMode(perm), nil
}
func DecodeBase64Content(content string) ([]byte, error) {
output, err := base64.StdEncoding.DecodeString(content)
if err != nil {
return nil, fmt.Errorf("Unable to decode base64: %v", err)
}
return output, nil
}
func DecodeGzipContent(content string) ([]byte, error) {
gzr, err := gzip.NewReader(bytes.NewReader([]byte(content)))
if err != nil {
return nil, fmt.Errorf("Unable to decode gzip: %v", err)
}
defer gzr.Close()
buf := new(bytes.Buffer)
buf.ReadFrom(gzr)
return buf.Bytes(), nil
}
func DecodeContent(content string, encoding string) ([]byte, error) {
switch encoding {
case "":
return []byte(content), nil
case "b64", "base64":
return DecodeBase64Content(content)
case "gz", "gzip":
return DecodeGzipContent(content)
case "gz+base64", "gzip+base64", "gz+b64", "gzip+b64":
gz, err := DecodeBase64Content(content)
if err != nil {
return nil, err
}
return DecodeGzipContent(string(gz))
}
return nil, fmt.Errorf("Unsupported encoding %s", encoding)
}
func WriteFile(f *File, root string) (string, error) {
fullpath := path.Join(root, f.Path)
dir := path.Dir(fullpath)
log.Printf("Writing file to %q", fullpath)
if f.Encoding != "" {
return "", fmt.Errorf("Unable to write file with encoding %s", f.Encoding)
content, err := DecodeContent(f.Content, f.Encoding)
if err != nil {
return "", fmt.Errorf("Unable to decode %s (%v)", f.Path, err)
}
if err := EnsureDirectoryExists(dir); err != nil {
@ -71,7 +125,7 @@ func WriteFile(f *File, root string) (string, error) {
return "", err
}
if err := ioutil.WriteFile(tmp.Name(), []byte(f.Content), perm); err != nil {
if err := ioutil.WriteFile(tmp.Name(), content, perm); err != nil {
return "", err
}

89
system/file_test.go
View File

@ -156,10 +156,97 @@ func TestWriteFileEncodedContent(t *testing.T) {
}
defer os.RemoveAll(dir)
//all of these decode to "bar"
content_tests := map[string]string{
"base64": "YmFy",
"b64": "YmFy",
"gz": "\x1f\x8b\x08\x08w\x14\x87T\x02\xffok\x00KJ,\x02\x00\xaa\x8c\xffv\x03\x00\x00\x00",
"gzip": "\x1f\x8b\x08\x08w\x14\x87T\x02\xffok\x00KJ,\x02\x00\xaa\x8c\xffv\x03\x00\x00\x00",
"gz+base64": "H4sIABMVh1QAA0tKLAIAqoz/dgMAAAA=",
"gzip+base64": "H4sIABMVh1QAA0tKLAIAqoz/dgMAAAA=",
"gz+b64": "H4sIABMVh1QAA0tKLAIAqoz/dgMAAAA=",
"gzip+b64": "H4sIABMVh1QAA0tKLAIAqoz/dgMAAAA=",
}
for encoding, content := range content_tests {
fullPath := path.Join(dir, encoding)
wf := File{config.File{
Path: encoding,
Encoding: encoding,
Content: content,
RawFilePermissions: "0644",
}}
path, err := WriteFile(&wf, dir)
if err != nil {
t.Fatalf("Processing of WriteFile failed: %v", err)
} else if path != fullPath {
t.Fatalf("WriteFile returned bad path: want %s, got %s", fullPath, path)
}
fi, err := os.Stat(fullPath)
if err != nil {
t.Fatalf("Unable to stat file: %v", err)
}
if fi.Mode() != os.FileMode(0644) {
t.Errorf("File has incorrect mode: %v", fi.Mode())
}
contents, err := ioutil.ReadFile(fullPath)
if err != nil {
t.Fatalf("Unable to read expected file: %v", err)
}
if string(contents) != "bar" {
t.Fatalf("File has incorrect contents: '%s'", contents)
}
}
}
func TestWriteFileInvalidEncodedContent(t *testing.T) {
dir, err := ioutil.TempDir(os.TempDir(), "coreos-cloudinit-")
if err != nil {
t.Fatalf("Unable to create tempdir: %v", err)
}
defer os.RemoveAll(dir)
content_encodings := []string{
"base64",
"b64",
"gz",
"gzip",
"gz+base64",
"gzip+base64",
"gz+b64",
"gzip+b64",
}
for _, encoding := range content_encodings {
wf := File{config.File{
Path: path.Join(dir, "tmp", "foo"),
Content: "@&*#%invalid data*@&^#*&",
Encoding: encoding,
}}
if _, err := WriteFile(&wf, dir); err == nil {
t.Fatalf("Expected error to be raised when writing file with encoding")
}
}
}
func TestWriteFileUnknownEncodedContent(t *testing.T) {
dir, err := ioutil.TempDir(os.TempDir(), "coreos-cloudinit-")
if err != nil {
t.Fatalf("Unable to create tempdir: %v", err)
}
defer os.RemoveAll(dir)
wf := File{config.File{
Path: path.Join(dir, "tmp", "foo"),
Content: "",
Encoding: "base64",
Encoding: "no-such-encoding",
}}
if _, err := WriteFile(&wf, dir); err == nil {