coreos-cloudinit: bump to v1.5.0

config: recognize Ignition configs and no-op

.travis.yml

@@ -4,10 +4,6 @@ matrix:
   include:
     - go: 1.4
       env: TOOLS_CMD=golang.org/x/tools/cmd
-    - go: 1.3
-      env: TOOLS_CMD=code.google.com/p/go.tools/cmd
-    - go: 1.2
-      env: TOOLS_CMD=code.google.com/p/go.tools/cmd
 
 install:
  - go get ${TOOLS_CMD}/cover

Documentation/cloud-config-deprecated.md

@@ -0,0 +1,38 @@
+# Deprecated Cloud-Config Features
+
+## Retrieving SSH Authorized Keys
+
+### From a GitHub User
+
+Using the `coreos-ssh-import-github` field, we can import public SSH keys from a GitHub user to use as authorized keys to a server.
+
+```yaml
+#cloud-config
+
+users:
+  - name: elroy
+    coreos-ssh-import-github: elroy
+```
+
+### From an HTTP Endpoint
+
+We can also pull public SSH keys from any HTTP endpoint which matches [GitHub's API response format](https://developer.github.com/v3/users/keys/#list-public-keys-for-a-user).
+For example, if you have an installation of GitHub Enterprise, you can provide a complete URL with an authentication token:
+
+```yaml
+#cloud-config
+
+users:
+  - name: elroy
+    coreos-ssh-import-url: https://github-enterprise.example.com/api/v3/users/elroy/keys?access_token=<TOKEN>
+```
+
+You can also specify any URL whose response matches the JSON format for public keys:
+
+```yaml
+#cloud-config
+
+users:
+  - name: elroy
+    coreos-ssh-import-url: https://example.com/public-keys
+```

Documentation/cloud-config.md

@@ -76,14 +76,16 @@ _Note: The `$private_ipv4` and `$public_ipv4` substitution variables referenced
 #### etcd2
 
 The `coreos.etcd2.*` parameters will be translated to a partial systemd unit acting as an etcd configuration file.
-If the platform environment supports the templating feature of coreos-cloudinit it is possible to automate etcd configuration with the `$private_ipv4` and `$public_ipv4` fields. For example, the following cloud-config document...
+If the platform environment supports the templating feature of coreos-cloudinit it is possible to automate etcd configuration with the `$private_ipv4` and `$public_ipv4` fields. When generating a [discovery token](https://discovery.etcd.io/new?size=3), set the `size` parameter, since etcd uses this to determine if all members have joined the cluster. After the cluster is bootstrapped, it can grow or shrink from this configured size.
+
+For example, the following cloud-config document...
 
 ```yaml
 #cloud-config
 
 coreos:
   etcd2:
-    # generate a new token for each unique cluster from https://discovery.etcd.io/new
+    # generate a new token for each unique cluster from https://discovery.etcd.io/new?size=3
     discovery: https://discovery.etcd.io/<token>
     # multi-region and multi-cloud deployments need to use $public_ipv4
     advertise-client-urls: http://$public_ipv4:2379
@@ -102,7 +104,7 @@ Environment="ETCD_DISCOVERY=https://discovery.etcd.io/<token>"
 Environment="ETCD_ADVERTISE_CLIENT_URLS=http://203.0.113.29:2379"
 Environment="ETCD_INITIAL_ADVERTISE_PEER_URLS=http://192.0.2.13:2380"
 Environment="ETCD_LISTEN_CLIENT_URLS=http://0.0.0.0:2379,http://0.0.0.0:4001"
-Environment="ETCD_LISTEN_PEERS_URLS=http://192.0.2.13:2380,http://192.0.2.13:7001"
+Environment="ETCD_LISTEN_PEER_URLS=http://192.0.2.13:2380,http://192.0.2.13:7001"
 ```
 
 For more information about the available configuration parameters, see the [etcd documentation][etcd-config].
@@ -332,9 +334,9 @@ All but the `passwd` and `ssh-authorized-keys` fields will be ignored if the use
 - **groups**: Add user to these additional groups
 - **no-user-group**: Boolean. Skip default group creation.
 - **ssh-authorized-keys**: List of public SSH keys to authorize for this user
-- **coreos-ssh-import-github**: Authorize SSH keys from GitHub user
-- **coreos-ssh-import-github-users**: Authorize SSH keys from a list of GitHub users
-- **coreos-ssh-import-url**: Authorize SSH keys imported from a url endpoint.
+- **coreos-ssh-import-github** [DEPRECATED]: Authorize SSH keys from GitHub user
+- **coreos-ssh-import-github-users** [DEPRECATED]: Authorize SSH keys from a list of GitHub users
+- **coreos-ssh-import-url** [DEPRECATED]: Authorize SSH keys imported from a url endpoint.
 - **system**: Create the user as a system user. No home directory will be created.
 - **no-log-init**: Boolean. Skip initialization of lastlog and faillog databases.
 - **shell**: User's login shell.
@@ -380,43 +382,6 @@ perl -e 'print crypt("password","\$6\$SALT\$") . "\n"'
 
 Using a higher number of rounds will help create more secure passwords, but given enough time, password hashes can be reversed.  On most RPM based distributions there is a tool called mkpasswd available in the `expect` package, but this does not handle "rounds" nor advanced hashing algorithms. 
 
-#### Retrieving SSH Authorized Keys
-
-##### From a GitHub User
-
-Using the `coreos-ssh-import-github` field, we can import public SSH keys from a GitHub user to use as authorized keys to a server.
-
-```yaml
-#cloud-config
-
-users:
-  - name: elroy
-    coreos-ssh-import-github: elroy
-```
-
-##### From an HTTP Endpoint
-
-We can also pull public SSH keys from any HTTP endpoint which matches [GitHub's API response format](https://developer.github.com/v3/users/keys/#list-public-keys-for-a-user).
-For example, if you have an installation of GitHub Enterprise, you can provide a complete URL with an authentication token:
-
-```yaml
-#cloud-config
-
-users:
-  - name: elroy
-    coreos-ssh-import-url: https://github-enterprise.example.com/api/v3/users/elroy/keys?access_token=<TOKEN>
-```
-
-You can also specify any URL whose response matches the JSON format for public keys:
-
-```yaml
-#cloud-config
-
-users:
-  - name: elroy
-    coreos-ssh-import-url: https://example.com/public-keys
-```
-
 ### write_files
 
 The `write_files` directive defines a set of files to create on the local filesystem.

config/ignition.go

@@ -0,0 +1,26 @@
+// Copyright 2015 CoreOS, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package config
+
+import (
+	"encoding/json"
+)
+
+func IsIgnitionConfig(userdata string) bool {
+	var cfg struct {
+		Version *int `json:"ignitionVersion" yaml:"ignition_version"`
+	}
+	return (json.Unmarshal([]byte(userdata), &cfg) == nil && cfg.Version != nil)
+}

config/user.go

@@ -18,9 +18,9 @@ type User struct {
 	Name                 string   `yaml:"name"`
 	PasswordHash         string   `yaml:"passwd"`
 	SSHAuthorizedKeys    []string `yaml:"ssh_authorized_keys"`
-	SSHImportGithubUser  string   `yaml:"coreos_ssh_import_github"`
-	SSHImportGithubUsers []string `yaml:"coreos_ssh_import_github_users"`
-	SSHImportURL         string   `yaml:"coreos_ssh_import_url"`
+	SSHImportGithubUser  string   `yaml:"coreos_ssh_import_github"       deprecated:"trying to fetch from a remote endpoint introduces too many intermittent errors"`
+	SSHImportGithubUsers []string `yaml:"coreos_ssh_import_github_users" deprecated:"trying to fetch from a remote endpoint introduces too many intermittent errors"`
+	SSHImportURL         string   `yaml:"coreos_ssh_import_url"          deprecated:"trying to fetch from a remote endpoint introduces too many intermittent errors"`
 	GECOS                string   `yaml:"gecos"`
 	Homedir              string   `yaml:"homedir"`
 	NoCreateHome         bool     `yaml:"no_create_home"`

config/validate/validate.go

@@ -40,6 +40,8 @@ func Validate(userdataBytes []byte) (Report, error) {
 		return Report{}, nil
 	case config.IsScript(string(userdataBytes)):
 		return Report{}, nil
+	case config.IsIgnitionConfig(string(userdataBytes)):
+		return Report{}, nil
 	case config.IsCloudConfig(string(userdataBytes)):
 		return validateCloudConfig(userdataBytes, Rules)
 	default:

config/validate/validate_test.go

@@ -111,6 +111,16 @@ func TestValidate(t *testing.T) {
 		{
 			config: "#!/bin/bash\necho hey",
 		},
+		{
+			config: "{}",
+			report: Report{entries: []Entry{{entryError, `must be "#cloud-config" or begin with "#!"`, 1}}},
+		},
+		{
+			config: `{"ignitionVersion":0}`,
+		},
+		{
+			config: `{"ignitionVersion":1}`,
+		},
 	}
 
 	for i, tt := range tests {

coreos-cloudinit.go

@@ -29,6 +29,7 @@ import (
 	"github.com/coreos/coreos-cloudinit/datasource/metadata/cloudsigma"
 	"github.com/coreos/coreos-cloudinit/datasource/metadata/digitalocean"
 	"github.com/coreos/coreos-cloudinit/datasource/metadata/ec2"
+	"github.com/coreos/coreos-cloudinit/datasource/metadata/packet"
 	"github.com/coreos/coreos-cloudinit/datasource/proc_cmdline"
 	"github.com/coreos/coreos-cloudinit/datasource/url"
 	"github.com/coreos/coreos-cloudinit/datasource/waagent"
@@ -39,7 +40,7 @@ import (
 )
 
 const (
-	version               = "1.4.0"
+	version               = "1.5.0"
 	datasourceInterval    = 100 * time.Millisecond
 	datasourceMaxInterval = 30 * time.Second
 	datasourceTimeout     = 5 * time.Minute
@@ -57,6 +58,7 @@ var (
 			ec2MetadataService          string
 			cloudSigmaMetadataService   bool
 			digitalOceanMetadataService string
+			packetMetadataService       string
 			url                         string
 			procCmdLine                 bool
 		}
@@ -78,6 +80,7 @@ func init() {
 	flag.StringVar(&flags.sources.ec2MetadataService, "from-ec2-metadata", "", "Download EC2 data from the provided url")
 	flag.BoolVar(&flags.sources.cloudSigmaMetadataService, "from-cloudsigma-metadata", false, "Download data from CloudSigma server context")
 	flag.StringVar(&flags.sources.digitalOceanMetadataService, "from-digitalocean-metadata", "", "Download DigitalOcean data from the provided url")
+	flag.StringVar(&flags.sources.packetMetadataService, "from-packet-metadata", "", "Download Packet data from metadata service")
 	flag.StringVar(&flags.sources.url, "from-url", "", "Download user-data from provided url")
 	flag.BoolVar(&flags.sources.procCmdLine, "from-proc-cmdline", false, fmt.Sprintf("Parse %s for '%s=<url>', using the cloud-config served by an HTTP GET to <url>", proc_cmdline.ProcCmdlineLocation, proc_cmdline.ProcCmdlineCloudConfigFlag))
 	flag.StringVar(&flags.oem, "oem", "", "Use the settings specific to the provided OEM")
@@ -109,6 +112,10 @@ var (
 		"cloudsigma": oemConfig{
 			"from-cloudsigma-metadata": "true",
 		},
+		"packet": oemConfig{
+			"from-packet-metadata": "https://metadata.packet.net/",
+			"convert-netconf":      "packet",
+		},
 	}
 )
 
@@ -139,14 +146,15 @@ func main() {
 	case "":
 	case "debian":
 	case "digitalocean":
+	case "packet":
 	default:
-		fmt.Printf("Invalid option to -convert-netconf: '%s'. Supported options: 'debian, digitalocean'\n", flags.convertNetconf)
+		fmt.Printf("Invalid option to -convert-netconf: '%s'. Supported options: 'debian, digitalocean, packet'\n", flags.convertNetconf)
 		os.Exit(2)
 	}
 
 	dss := getDatasources()
 	if len(dss) == 0 {
-		fmt.Println("Provide at least one of --from-file, --from-configdrive, --from-ec2-metadata, --from-cloudsigma-metadata, --from-url or --from-proc-cmdline")
+		fmt.Println("Provide at least one of --from-file, --from-configdrive, --from-ec2-metadata, --from-cloudsigma-metadata, --from-packet-metadata, --from-url or --from-proc-cmdline")
 		os.Exit(2)
 	}
 
@@ -192,16 +200,20 @@ func main() {
 
 	var ccu *config.CloudConfig
 	var script *config.Script
-	if ud, err := initialize.ParseUserData(userdata); err != nil {
-		fmt.Printf("Failed to parse user-data: %v\nContinuing...\n", err)
-		failure = true
-	} else {
+	switch ud, err := initialize.ParseUserData(userdata); err {
+	case initialize.ErrIgnitionConfig:
+		fmt.Printf("Detected an Ignition config. Exiting...")
+		os.Exit(0)
+	case nil:
 		switch t := ud.(type) {
 		case *config.CloudConfig:
 			ccu = t
 		case *config.Script:
 			script = t
 		}
+	default:
+		fmt.Printf("Failed to parse user-data: %v\nContinuing...\n", err)
+		failure = true
 	}
 
 	fmt.Println("Merging cloud-config from meta-data and user-data")
@@ -215,6 +227,8 @@ func main() {
 			ifaces, err = network.ProcessDebianNetconf(metadata.NetworkConfig)
 		case "digitalocean":
 			ifaces, err = network.ProcessDigitalOceanNetconf(metadata.NetworkConfig)
+		case "packet":
+			ifaces, err = network.ProcessPacketNetconf(metadata.NetworkConfig)
 		default:
 			err = fmt.Errorf("Unsupported network config format %q", flags.convertNetconf)
 		}
@@ -290,6 +304,9 @@ func getDatasources() []datasource.Datasource {
 	if flags.sources.waagent != "" {
 		dss = append(dss, waagent.NewDatasource(flags.sources.waagent))
 	}
+	if flags.sources.packetMetadataService != "" {
+		dss = append(dss, packet.NewDatasource(flags.sources.packetMetadataService))
+	}
 	if flags.sources.procCmdLine {
 		dss = append(dss, proc_cmdline.NewDatasource())
 	}

datasource/metadata/packet/metadata.go

@@ -0,0 +1,106 @@
+// Copyright 2015 CoreOS, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package packet
+
+import (
+	"encoding/json"
+	"net"
+	"strconv"
+
+	"github.com/coreos/coreos-cloudinit/datasource"
+	"github.com/coreos/coreos-cloudinit/datasource/metadata"
+)
+
+const (
+	DefaultAddress = "https://metadata.packet.net/"
+	apiVersion     = ""
+	userdataUrl    = "userdata"
+	metadataPath   = "metadata"
+)
+
+type Netblock struct {
+	Address       net.IP `json:"address"`
+	Cidr          int    `json:"cidr"`
+	Netmask       net.IP `json:"netmask"`
+	Gateway       net.IP `json:"gateway"`
+	AddressFamily int    `json:"address_family"`
+	Public        bool   `json:"public"`
+}
+
+type Nic struct {
+	Name string `json:"name"`
+	Mac  string `json:"mac"`
+}
+
+type NetworkData struct {
+	Interfaces []Nic      `json:"interfaces"`
+	Netblocks  []Netblock `json:"addresses"`
+	DNS        []net.IP   `json:"dns"`
+}
+
+// Metadata that will be pulled from the https://metadata.packet.net/metadata only. We have the opportunity to add more later.
+type Metadata struct {
+	Hostname    string      `json:"hostname"`
+	SSHKeys     []string    `json:"ssh_keys"`
+	NetworkData NetworkData `json:"network"`
+}
+
+type metadataService struct {
+	metadata.MetadataService
+}
+
+func NewDatasource(root string) *metadataService {
+	return &metadataService{MetadataService: metadata.NewDatasource(root, apiVersion, userdataUrl, metadataPath)}
+}
+
+func (ms *metadataService) FetchMetadata() (metadata datasource.Metadata, err error) {
+	var data []byte
+	var m Metadata
+
+	if data, err = ms.FetchData(ms.MetadataUrl()); err != nil || len(data) == 0 {
+		return
+	}
+
+	if err = json.Unmarshal(data, &m); err != nil {
+		return
+	}
+
+	if len(m.NetworkData.Netblocks) > 0 {
+		for _, Netblock := range m.NetworkData.Netblocks {
+			if Netblock.AddressFamily == 4 {
+				if Netblock.Public == true {
+					metadata.PublicIPv4 = Netblock.Address
+				} else {
+					metadata.PrivateIPv4 = Netblock.Address
+				}
+			} else {
+				metadata.PublicIPv6 = Netblock.Address
+			}
+		}
+	}
+	metadata.Hostname = m.Hostname
+	metadata.SSHPublicKeys = map[string]string{}
+	for i, key := range m.SSHKeys {
+		metadata.SSHPublicKeys[strconv.Itoa(i)] = key
+	}
+
+	metadata.NetworkConfig, err = json.Marshal(m.NetworkData)
+
+	return
+}
+
+func (ms metadataService) Type() string {
+	return "packet-metadata-service"
+}

initialize/user_data.go

@@ -21,6 +21,10 @@ import (
 	"github.com/coreos/coreos-cloudinit/config"
 )
 
+var (
+	ErrIgnitionConfig = errors.New("not a config (found Ignition)")
+)
+
 func ParseUserData(contents string) (interface{}, error) {
 	if len(contents) == 0 {
 		return nil, nil
@@ -33,6 +37,8 @@ func ParseUserData(contents string) (interface{}, error) {
 	case config.IsCloudConfig(contents):
 		log.Printf("Parsing user-data as cloud-config")
 		return config.NewCloudConfig(contents)
+	case config.IsIgnitionConfig(contents):
+		return nil, ErrIgnitionConfig
 	default:
 		return nil, errors.New("Unrecognized user-data format")
 	}

network/interface.go

@@ -130,7 +130,17 @@ type bondInterface struct {
 }
 
 func (b *bondInterface) Netdev() string {
-	return fmt.Sprintf("[NetDev]\nKind=bond\nName=%s\n", b.name)
+	config := fmt.Sprintf("[NetDev]\nKind=bond\nName=%s\n", b.name)
+	if b.hwaddr != nil {
+		config += fmt.Sprintf("MACAddress=%s\n", b.hwaddr.String())
+	}
+
+	config += fmt.Sprintf("\n[Bond]\n")
+	for _, name := range sortedKeys(b.options) {
+		config += fmt.Sprintf("%s=%s\n", name, b.options[name])
+	}
+
+	return config
 }
 
 func (b *bondInterface) Type() string {

network/interface_test.go

@@ -52,7 +52,7 @@ func TestInterfaceGenerators(t *testing.T) {
 		},
 		{
 			name:    "testname",
-			netdev:  "[NetDev]\nKind=bond\nName=testname\n",
+			netdev:  "[NetDev]\nKind=bond\nName=testname\n\n[Bond]\n",
 			network: "[Match]\nName=testname\n\n[Network]\nBond=testbond1\nVLAN=testvlan1\nVLAN=testvlan2\nDHCP=true\n",
 			kind:    "bond",
 			iface: &bondInterface{logicalInterface: logicalInterface{

network/packet.go

@@ -0,0 +1,133 @@
+// Copyright 2015 CoreOS, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package network
+
+import (
+	"encoding/json"
+	"net"
+
+	"github.com/coreos/coreos-cloudinit/datasource/metadata/packet"
+)
+
+func ProcessPacketNetconf(config []byte) ([]InterfaceGenerator, error) {
+	var netdata packet.NetworkData
+	if err := json.Unmarshal(config, &netdata); err != nil {
+		return nil, err
+	}
+
+	var nameservers []net.IP
+	if netdata.DNS != nil {
+		nameservers = netdata.DNS
+	} else {
+		nameservers = append(nameservers, net.ParseIP("8.8.8.8"), net.ParseIP("8.8.4.4"))
+	}
+
+	generators, err := parseNetwork(netdata, nameservers)
+	if err != nil {
+		return nil, err
+	}
+
+	return generators, nil
+}
+
+func parseNetwork(netdata packet.NetworkData, nameservers []net.IP) ([]InterfaceGenerator, error) {
+	var interfaces []InterfaceGenerator
+	var addresses []net.IPNet
+	var routes []route
+	for _, netblock := range netdata.Netblocks {
+		addresses = append(addresses, net.IPNet{
+			IP:   netblock.Address,
+			Mask: net.IPMask(netblock.Netmask),
+		})
+		if netblock.Public == false {
+			routes = append(routes, route{
+				destination: net.IPNet{
+					IP:   net.IPv4(10, 0, 0, 0),
+					Mask: net.IPv4Mask(255, 0, 0, 0),
+				},
+				gateway: netblock.Gateway,
+			})
+		} else {
+			if netblock.AddressFamily == 4 {
+				routes = append(routes, route{
+					destination: net.IPNet{
+						IP:   net.IPv4zero,
+						Mask: net.IPMask(net.IPv4zero),
+					},
+					gateway: netblock.Gateway,
+				})
+			} else {
+				routes = append(routes, route{
+					destination: net.IPNet{
+						IP:   net.IPv6zero,
+						Mask: net.IPMask(net.IPv6zero),
+					},
+					gateway: netblock.Gateway,
+				})
+			}
+		}
+	}
+
+	bond := bondInterface{
+		logicalInterface: logicalInterface{
+			name: "bond0",
+			config: configMethodStatic{
+				addresses:   addresses,
+				nameservers: nameservers,
+				routes:      routes,
+			},
+		},
+		options: map[string]string{
+			"Mode":             "802.3ad",
+			"LACPTransmitRate": "fast",
+			"MIIMonitorSec":    ".2",
+			"UpDelaySec":       ".2",
+			"DownDelaySec":     ".2",
+		},
+	}
+
+	for _, iface := range netdata.Interfaces {
+		if iface.Name != "chassis0" && iface.Name != "ipmi0" {
+			bond.slaves = append(bond.slaves, iface.Name)
+			if iface.Name == "enp1s0f0" {
+				bond.hwaddr, _ = net.ParseMAC(iface.Mac)
+			}
+		}
+	}
+
+	for _, iface := range netdata.Interfaces {
+		if iface.Name != "chassis0" && iface.Name != "ipmi0" {
+			p := physicalInterface{
+				logicalInterface: logicalInterface{
+					name: iface.Name,
+					config: configMethodStatic{
+						nameservers: nameservers,
+					},
+					children: []networkInterface{&bond},
+				},
+			}
+
+			if iface.Name == "enp1s0f0" {
+				p.configDepth = 20
+			}
+
+			interfaces = append(interfaces, &p)
+		}
+	}
+
+	interfaces = append(interfaces, &bond)
+
+	return interfaces, nil
+}