From 081f77a1025d39f4dcfdb608bd43beb486ffcbb1 Mon Sep 17 00:00:00 2001 From: Sam Tresler Date: Tue, 23 Jun 2015 11:21:26 -0400 Subject: [PATCH 1/9] Persisting bond options to the netdev file, and updating the test. --- network/interface.go | 12 +++++++++++- network/interface_test.go | 2 +- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/network/interface.go b/network/interface.go index 877cb9e..73a83cb 100644 --- a/network/interface.go +++ b/network/interface.go @@ -130,7 +130,17 @@ type bondInterface struct { } func (b *bondInterface) Netdev() string { - return fmt.Sprintf("[NetDev]\nKind=bond\nName=%s\n", b.name) + config := fmt.Sprintf("[NetDev]\nKind=bond\nName=%s\n", b.name) + if b.hwaddr != nil { + config += fmt.Sprintf("MACAddress=%s\n", b.hwaddr.String()) + } + + config += fmt.Sprintf("\n[Bond]\n") + for _, name := range sortedKeys(b.options) { + config += fmt.Sprintf("%s=%s\n", name, b.options[name]) + } + + return config } func (b *bondInterface) Type() string { diff --git a/network/interface_test.go b/network/interface_test.go index df6eea3..5cafc93 100644 --- a/network/interface_test.go +++ b/network/interface_test.go @@ -52,7 +52,7 @@ func TestInterfaceGenerators(t *testing.T) { }, { name: "testname", - netdev: "[NetDev]\nKind=bond\nName=testname\n", + netdev: "[NetDev]\nKind=bond\nName=testname\n\n[Bond]\n", network: "[Match]\nName=testname\n\n[Network]\nBond=testbond1\nVLAN=testvlan1\nVLAN=testvlan2\nDHCP=true\n", kind: "bond", iface: &bondInterface{logicalInterface: logicalInterface{ From 837d3d3622a3007f6923fbb6cb373b301f262382 Mon Sep 17 00:00:00 2001 From: Sam Tresler Date: Fri, 10 Jul 2015 15:13:57 -0400 Subject: [PATCH 2/9] datasource: add packethost metadata --- coreos-cloudinit.go | 17 +++- datasource/metadata/packet/metadata.go | 106 ++++++++++++++++++++ network/packet.go | 133 +++++++++++++++++++++++++ 3 files changed, 254 insertions(+), 2 deletions(-) create mode 100644 datasource/metadata/packet/metadata.go create mode 100644 network/packet.go diff --git a/coreos-cloudinit.go b/coreos-cloudinit.go index f147fb9..e29ff7e 100644 --- a/coreos-cloudinit.go +++ b/coreos-cloudinit.go @@ -29,6 +29,7 @@ import ( "github.com/coreos/coreos-cloudinit/datasource/metadata/cloudsigma" "github.com/coreos/coreos-cloudinit/datasource/metadata/digitalocean" "github.com/coreos/coreos-cloudinit/datasource/metadata/ec2" + "github.com/coreos/coreos-cloudinit/datasource/metadata/packet" "github.com/coreos/coreos-cloudinit/datasource/proc_cmdline" "github.com/coreos/coreos-cloudinit/datasource/url" "github.com/coreos/coreos-cloudinit/datasource/waagent" @@ -57,6 +58,7 @@ var ( ec2MetadataService string cloudSigmaMetadataService bool digitalOceanMetadataService string + packetMetadataService string url string procCmdLine bool } @@ -78,6 +80,7 @@ func init() { flag.StringVar(&flags.sources.ec2MetadataService, "from-ec2-metadata", "", "Download EC2 data from the provided url") flag.BoolVar(&flags.sources.cloudSigmaMetadataService, "from-cloudsigma-metadata", false, "Download data from CloudSigma server context") flag.StringVar(&flags.sources.digitalOceanMetadataService, "from-digitalocean-metadata", "", "Download DigitalOcean data from the provided url") + flag.StringVar(&flags.sources.packetMetadataService, "from-packet-metadata", "", "Download Packet data from metadata service") flag.StringVar(&flags.sources.url, "from-url", "", "Download user-data from provided url") flag.BoolVar(&flags.sources.procCmdLine, "from-proc-cmdline", false, fmt.Sprintf("Parse %s for '%s=', using the cloud-config served by an HTTP GET to ", proc_cmdline.ProcCmdlineLocation, proc_cmdline.ProcCmdlineCloudConfigFlag)) flag.StringVar(&flags.oem, "oem", "", "Use the settings specific to the provided OEM") @@ -109,6 +112,10 @@ var ( "cloudsigma": oemConfig{ "from-cloudsigma-metadata": "true", }, + "packet": oemConfig{ + "from-packet-metadata": "https://metadata.packet.net/", + "convert-netconf": "packet", + }, } ) @@ -139,14 +146,15 @@ func main() { case "": case "debian": case "digitalocean": + case "packet": default: - fmt.Printf("Invalid option to -convert-netconf: '%s'. Supported options: 'debian, digitalocean'\n", flags.convertNetconf) + fmt.Printf("Invalid option to -convert-netconf: '%s'. Supported options: 'debian, digitalocean, packet'\n", flags.convertNetconf) os.Exit(2) } dss := getDatasources() if len(dss) == 0 { - fmt.Println("Provide at least one of --from-file, --from-configdrive, --from-ec2-metadata, --from-cloudsigma-metadata, --from-url or --from-proc-cmdline") + fmt.Println("Provide at least one of --from-file, --from-configdrive, --from-ec2-metadata, --from-cloudsigma-metadata, --from-packet-metadata, --from-url or --from-proc-cmdline") os.Exit(2) } @@ -215,6 +223,8 @@ func main() { ifaces, err = network.ProcessDebianNetconf(metadata.NetworkConfig) case "digitalocean": ifaces, err = network.ProcessDigitalOceanNetconf(metadata.NetworkConfig) + case "packet": + ifaces, err = network.ProcessPacketNetconf(metadata.NetworkConfig) default: err = fmt.Errorf("Unsupported network config format %q", flags.convertNetconf) } @@ -290,6 +300,9 @@ func getDatasources() []datasource.Datasource { if flags.sources.waagent != "" { dss = append(dss, waagent.NewDatasource(flags.sources.waagent)) } + if flags.sources.packetMetadataService != "" { + dss = append(dss, packet.NewDatasource(flags.sources.packetMetadataService)) + } if flags.sources.procCmdLine { dss = append(dss, proc_cmdline.NewDatasource()) } diff --git a/datasource/metadata/packet/metadata.go b/datasource/metadata/packet/metadata.go new file mode 100644 index 0000000..3928a2c --- /dev/null +++ b/datasource/metadata/packet/metadata.go @@ -0,0 +1,106 @@ +// Copyright 2015 CoreOS, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package packet + +import ( + "encoding/json" + "net" + "strconv" + + "github.com/coreos/coreos-cloudinit/datasource" + "github.com/coreos/coreos-cloudinit/datasource/metadata" +) + +const ( + DefaultAddress = "https://metadata.packet.net/" + apiVersion = "" + userdataUrl = "userdata" + metadataPath = "metadata" +) + +type Netblock struct { + Address net.IP `json:"address"` + Cidr int `json:"cidr"` + Netmask net.IP `json:"netmask"` + Gateway net.IP `json:"gateway"` + AddressFamily int `json:"address_family"` + Public bool `json:"public"` +} + +type Nic struct { + Name string `json:"name"` + Mac string `json:"mac"` +} + +type NetworkData struct { + Interfaces []Nic `json:"interfaces"` + Netblocks []Netblock `json:"addresses"` + DNS []net.IP `json:"dns"` +} + +// Metadata that will be pulled from the https://metadata.packet.net/metadata only. We have the opportunity to add more later. +type Metadata struct { + Hostname string `json:"hostname"` + SSHKeys []string `json:"ssh_keys"` + NetworkData NetworkData `json:"network"` +} + +type metadataService struct { + metadata.MetadataService +} + +func NewDatasource(root string) *metadataService { + return &metadataService{MetadataService: metadata.NewDatasource(root, apiVersion, userdataUrl, metadataPath)} +} + +func (ms *metadataService) FetchMetadata() (metadata datasource.Metadata, err error) { + var data []byte + var m Metadata + + if data, err = ms.FetchData(ms.MetadataUrl()); err != nil || len(data) == 0 { + return + } + + if err = json.Unmarshal(data, &m); err != nil { + return + } + + if len(m.NetworkData.Netblocks) > 0 { + for _, Netblock := range m.NetworkData.Netblocks { + if Netblock.AddressFamily == 4 { + if Netblock.Public == true { + metadata.PublicIPv4 = Netblock.Address + } else { + metadata.PrivateIPv4 = Netblock.Address + } + } else { + metadata.PublicIPv6 = Netblock.Address + } + } + } + metadata.Hostname = m.Hostname + metadata.SSHPublicKeys = map[string]string{} + for i, key := range m.SSHKeys { + metadata.SSHPublicKeys[strconv.Itoa(i)] = key + } + + metadata.NetworkConfig, err = json.Marshal(m.NetworkData) + + return +} + +func (ms metadataService) Type() string { + return "packet-metadata-service" +} diff --git a/network/packet.go b/network/packet.go new file mode 100644 index 0000000..e763049 --- /dev/null +++ b/network/packet.go @@ -0,0 +1,133 @@ +// Copyright 2015 CoreOS, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package network + +import ( + "encoding/json" + "net" + + "github.com/coreos/coreos-cloudinit/datasource/metadata/packet" +) + +func ProcessPacketNetconf(config []byte) ([]InterfaceGenerator, error) { + var netdata packet.NetworkData + if err := json.Unmarshal(config, &netdata); err != nil { + return nil, err + } + + var nameservers []net.IP + if netdata.DNS != nil { + nameservers = netdata.DNS + } else { + nameservers = append(nameservers, net.ParseIP("8.8.8.8"), net.ParseIP("8.8.4.4")) + } + + generators, err := parseNetwork(netdata, nameservers) + if err != nil { + return nil, err + } + + return generators, nil +} + +func parseNetwork(netdata packet.NetworkData, nameservers []net.IP) ([]InterfaceGenerator, error) { + var interfaces []InterfaceGenerator + var addresses []net.IPNet + var routes []route + for _, netblock := range netdata.Netblocks { + addresses = append(addresses, net.IPNet{ + IP: netblock.Address, + Mask: net.IPMask(netblock.Netmask), + }) + if netblock.Public == false { + routes = append(routes, route{ + destination: net.IPNet{ + IP: net.IPv4(10, 0, 0, 0), + Mask: net.IPv4Mask(255, 0, 0, 0), + }, + gateway: netblock.Gateway, + }) + } else { + if netblock.AddressFamily == 4 { + routes = append(routes, route{ + destination: net.IPNet{ + IP: net.IPv4zero, + Mask: net.IPMask(net.IPv4zero), + }, + gateway: netblock.Gateway, + }) + } else { + routes = append(routes, route{ + destination: net.IPNet{ + IP: net.IPv6zero, + Mask: net.IPMask(net.IPv6zero), + }, + gateway: netblock.Gateway, + }) + } + } + } + + bond := bondInterface{ + logicalInterface: logicalInterface{ + name: "bond0", + config: configMethodStatic{ + addresses: addresses, + nameservers: nameservers, + routes: routes, + }, + }, + options: map[string]string{ + "Mode": "802.3ad", + "LACPTransmitRate": "fast", + "MIIMonitorSec": ".2", + "UpDelaySec": ".2", + "DownDelaySec": ".2", + }, + } + + for _, iface := range netdata.Interfaces { + if iface.Name != "chassis0" && iface.Name != "ipmi0" { + bond.slaves = append(bond.slaves, iface.Name) + if iface.Name == "enp1s0f0" { + bond.hwaddr, _ = net.ParseMAC(iface.Mac) + } + } + } + + for _, iface := range netdata.Interfaces { + if iface.Name != "chassis0" && iface.Name != "ipmi0" { + p := physicalInterface{ + logicalInterface: logicalInterface{ + name: iface.Name, + config: configMethodStatic{ + nameservers: nameservers, + }, + children: []networkInterface{&bond}, + }, + } + + if iface.Name == "enp1s0f0" { + p.configDepth = 20 + } + + interfaces = append(interfaces, &p) + } + } + + interfaces = append(interfaces, &bond) + + return interfaces, nil +} From c6400f77518455c31e302c1977758c81f4911596 Mon Sep 17 00:00:00 2001 From: Alex Crawford Date: Mon, 6 Jul 2015 17:37:02 -0700 Subject: [PATCH 3/9] config: recognize Ignition configs and no-op --- config/ignition.go | 26 ++++++++++++++++++++++++++ config/validate/validate.go | 2 ++ config/validate/validate_test.go | 10 ++++++++++ coreos-cloudinit.go | 12 ++++++++---- initialize/user_data.go | 6 ++++++ 5 files changed, 52 insertions(+), 4 deletions(-) create mode 100644 config/ignition.go diff --git a/config/ignition.go b/config/ignition.go new file mode 100644 index 0000000..413a636 --- /dev/null +++ b/config/ignition.go @@ -0,0 +1,26 @@ +// Copyright 2015 CoreOS, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +import ( + "encoding/json" +) + +func IsIgnitionConfig(userdata string) bool { + var cfg struct { + Version *int `json:"ignitionVersion" yaml:"ignition_version"` + } + return (json.Unmarshal([]byte(userdata), &cfg) == nil && cfg.Version != nil) +} diff --git a/config/validate/validate.go b/config/validate/validate.go index e1b7a9f..d8bd1e5 100644 --- a/config/validate/validate.go +++ b/config/validate/validate.go @@ -40,6 +40,8 @@ func Validate(userdataBytes []byte) (Report, error) { return Report{}, nil case config.IsScript(string(userdataBytes)): return Report{}, nil + case config.IsIgnitionConfig(string(userdataBytes)): + return Report{}, nil case config.IsCloudConfig(string(userdataBytes)): return validateCloudConfig(userdataBytes, Rules) default: diff --git a/config/validate/validate_test.go b/config/validate/validate_test.go index dee79ad..0bb3a39 100644 --- a/config/validate/validate_test.go +++ b/config/validate/validate_test.go @@ -111,6 +111,16 @@ func TestValidate(t *testing.T) { { config: "#!/bin/bash\necho hey", }, + { + config: "{}", + report: Report{entries: []Entry{{entryError, `must be "#cloud-config" or begin with "#!"`, 1}}}, + }, + { + config: `{"ignitionVersion":0}`, + }, + { + config: `{"ignitionVersion":1}`, + }, } for i, tt := range tests { diff --git a/coreos-cloudinit.go b/coreos-cloudinit.go index e29ff7e..ac98107 100644 --- a/coreos-cloudinit.go +++ b/coreos-cloudinit.go @@ -200,16 +200,20 @@ func main() { var ccu *config.CloudConfig var script *config.Script - if ud, err := initialize.ParseUserData(userdata); err != nil { - fmt.Printf("Failed to parse user-data: %v\nContinuing...\n", err) - failure = true - } else { + switch ud, err := initialize.ParseUserData(userdata); err { + case initialize.ErrIgnitionConfig: + fmt.Printf("Detected an Ignition config. Exiting...") + os.Exit(0) + case nil: switch t := ud.(type) { case *config.CloudConfig: ccu = t case *config.Script: script = t } + default: + fmt.Printf("Failed to parse user-data: %v\nContinuing...\n", err) + failure = true } fmt.Println("Merging cloud-config from meta-data and user-data") diff --git a/initialize/user_data.go b/initialize/user_data.go index 170efaa..c728d64 100644 --- a/initialize/user_data.go +++ b/initialize/user_data.go @@ -21,6 +21,10 @@ import ( "github.com/coreos/coreos-cloudinit/config" ) +var ( + ErrIgnitionConfig = errors.New("not a config (found Ignition)") +) + func ParseUserData(contents string) (interface{}, error) { if len(contents) == 0 { return nil, nil @@ -33,6 +37,8 @@ func ParseUserData(contents string) (interface{}, error) { case config.IsCloudConfig(contents): log.Printf("Parsing user-data as cloud-config") return config.NewCloudConfig(contents) + case config.IsIgnitionConfig(contents): + return nil, ErrIgnitionConfig default: return nil, errors.New("Unrecognized user-data format") } From f36821f7cec9108ad8ec9244bec63a4346863408 Mon Sep 17 00:00:00 2001 From: Alex Crawford Date: Tue, 14 Jul 2015 11:54:56 -0700 Subject: [PATCH 4/9] coreos-cloudinit: bump to v1.5.0 --- coreos-cloudinit.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/coreos-cloudinit.go b/coreos-cloudinit.go index ac98107..7018145 100644 --- a/coreos-cloudinit.go +++ b/coreos-cloudinit.go @@ -40,7 +40,7 @@ import ( ) const ( - version = "1.4.1+git" + version = "1.5.0" datasourceInterval = 100 * time.Millisecond datasourceMaxInterval = 30 * time.Second datasourceTimeout = 5 * time.Minute From ba83b2871fffe35df74355a793624fc78ecbad05 Mon Sep 17 00:00:00 2001 From: Alex Crawford Date: Tue, 14 Jul 2015 11:55:39 -0700 Subject: [PATCH 5/9] coreos-cloudinit: bump to v1.5.0+git --- coreos-cloudinit.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/coreos-cloudinit.go b/coreos-cloudinit.go index 7018145..3d91614 100644 --- a/coreos-cloudinit.go +++ b/coreos-cloudinit.go @@ -40,7 +40,7 @@ import ( ) const ( - version = "1.5.0" + version = "1.5.0+git" datasourceInterval = 100 * time.Millisecond datasourceMaxInterval = 30 * time.Second datasourceTimeout = 5 * time.Minute From eb8fc045eef7f8acadcf35a9e2797c61bb4f7553 Mon Sep 17 00:00:00 2001 From: Matt Dainty Date: Mon, 20 Jul 2015 10:27:02 +0100 Subject: [PATCH 6/9] Add note for creating config-drive ISO on OS X --- Documentation/config-drive.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Documentation/config-drive.md b/Documentation/config-drive.md index 2c7995f..b3bc884 100644 --- a/Documentation/config-drive.md +++ b/Documentation/config-drive.md @@ -21,6 +21,12 @@ mkisofs -R -V config-2 -o configdrive.iso /tmp/new-drive rm -r /tmp/new-drive ``` +If on OS X, replace the `mkisofs` invocation with: + +```sh +hdiutil makehybrid -iso -joliet -default-volume-name config-2 -o configdrive.iso /tmp/new-drive +``` + ## QEMU virtfs One exception to the above, when using QEMU it is possible to skip creating an From 91fe744bd2a64f64e68034dd5bd6de3100168385 Mon Sep 17 00:00:00 2001 From: Alex Crawford Date: Tue, 21 Jul 2015 14:20:10 -0700 Subject: [PATCH 7/9] config: specific valid values for ETCD_PROXY --- config/etcd2.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/etcd2.go b/config/etcd2.go index 1b809b4..fd3a432 100644 --- a/config/etcd2.go +++ b/config/etcd2.go @@ -39,6 +39,6 @@ type Etcd2 struct { PeerCAFile string `yaml:"peer_ca_file" env:"ETCD_PEER_CA_FILE"` PeerCertFile string `yaml:"peer_cert_file" env:"ETCD_PEER_CERT_FILE"` PeerKeyFile string `yaml:"peer_key_file" env:"ETCD_PEER_KEY_FILE"` - Proxy string `yaml:"proxy" env:"ETCD_PROXY"` + Proxy string `yaml:"proxy" env:"ETCD_PROXY" valid:"^(on|off|readonly)$"` SnapshotCount int `yaml:"snapshot_count" env:"ETCD_SNAPSHOTCOUNT"` } From 7ab84601c36e9545190f9330ef2ebba683aee77a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Moreira?= Date: Wed, 22 Jul 2015 22:54:03 +0100 Subject: [PATCH 8/9] Add PublicIP opt --- config/flannel.go | 1 + 1 file changed, 1 insertion(+) diff --git a/config/flannel.go b/config/flannel.go index 043afc9..7a953f8 100644 --- a/config/flannel.go +++ b/config/flannel.go @@ -23,4 +23,5 @@ type Flannel struct { IPMasq string `yaml:"ip_masq" env:"FLANNELD_IP_MASQ"` SubnetFile string `yaml:"subnet_file" env:"FLANNELD_SUBNET_FILE"` Iface string `yaml:"interface" env:"FLANNELD_IFACE"` + PublicIP string `yaml:"public_ip" env:"FLANNELD_PUBLIC_IP"` } From 43c6da06a521fde54990c5cc37a94b814d610c2f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jos=C3=A9=20Moreira?= Date: Thu, 23 Jul 2015 17:04:37 +0100 Subject: [PATCH 9/9] add public_ip opt on cloud-config.md --- Documentation/cloud-config.md | 1 + 1 file changed, 1 insertion(+) diff --git a/Documentation/cloud-config.md b/Documentation/cloud-config.md index ffce8fc..68c701f 100644 --- a/Documentation/cloud-config.md +++ b/Documentation/cloud-config.md @@ -169,6 +169,7 @@ List of flannel configuration parameters: - **ip_masq**: Install IP masquerade rules for traffic outside of flannel subnet - **subnet_file**: Path to flannel subnet file to write out - **interface**: Interface (name or IP) that should be used for inter-host communication +- **public_ip**: IP accessible by other nodes for inter-host communication [flannel-readme]: https://github.com/coreos/flannel/blob/master/README.md