From 259c7e1fe2b993a41863ab4593bfae6fc98b83f7 Mon Sep 17 00:00:00 2001 From: Alex Crawford Date: Wed, 18 Jun 2014 11:40:13 -0700 Subject: [PATCH 1/4] fix(sshKeyName): Use the SSH key name provided --- coreos-cloudinit.go | 2 +- initialize/env.go | 4 ++-- initialize/env_test.go | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/coreos-cloudinit.go b/coreos-cloudinit.go index 79d4c15..8c55202 100644 --- a/coreos-cloudinit.go +++ b/coreos-cloudinit.go @@ -89,7 +89,7 @@ func main() { } } - env := initialize.NewEnvironment("/", workspace) + env := initialize.NewEnvironment("/", workspace, sshKeyName) if len(userdataBytes) > 0 { if err := processUserdata(string(userdataBytes), env); err != nil { fmt.Printf("Failed resolving user-data: %v\n", err) diff --git a/initialize/env.go b/initialize/env.go index 7460c2d..b0ab3c1 100644 --- a/initialize/env.go +++ b/initialize/env.go @@ -15,12 +15,12 @@ type Environment struct { substitutions map[string]string } -func NewEnvironment(root, workspace string) *Environment { +func NewEnvironment(root, workspace, sshKeyName string) *Environment { substitutions := map[string]string{ "$public_ipv4": os.Getenv("COREOS_PUBLIC_IPV4"), "$private_ipv4": os.Getenv("COREOS_PRIVATE_IPV4"), } - return &Environment{root, workspace, DefaultSSHKeyName, substitutions} + return &Environment{root, workspace, sshKeyName, substitutions} } func (self *Environment) Workspace() string { diff --git a/initialize/env_test.go b/initialize/env_test.go index 432663b..f10a054 100644 --- a/initialize/env_test.go +++ b/initialize/env_test.go @@ -8,7 +8,7 @@ import ( func TestEnvironmentApply(t *testing.T) { os.Setenv("COREOS_PUBLIC_IPV4", "192.0.2.3") os.Setenv("COREOS_PRIVATE_IPV4", "192.0.2.203") - env := NewEnvironment("./", "./") + env := NewEnvironment("./", "./", "") input := `[Service] ExecStart=/usr/bin/echo "$public_ipv4" ExecStop=/usr/bin/echo $private_ipv4 From 29ed6b38bd4515c5862f7f1316cba21f4180e963 Mon Sep 17 00:00:00 2001 From: Alex Crawford Date: Wed, 18 Jun 2014 11:36:06 -0700 Subject: [PATCH 2/4] refactor(env): Add the config root and netconf type to datasource and env --- coreos-cloudinit.go | 2 +- datasource/configdrive.go | 24 ++++++++++++++++-------- datasource/datasource.go | 1 + datasource/file.go | 4 ++++ datasource/metadata_service.go | 4 ++++ datasource/proc_cmdline.go | 6 +++++- initialize/env.go | 14 ++++++++++++-- initialize/env_test.go | 2 +- 8 files changed, 44 insertions(+), 13 deletions(-) diff --git a/coreos-cloudinit.go b/coreos-cloudinit.go index 8c55202..99ceaac 100644 --- a/coreos-cloudinit.go +++ b/coreos-cloudinit.go @@ -89,7 +89,7 @@ func main() { } } - env := initialize.NewEnvironment("/", workspace, sshKeyName) + env := initialize.NewEnvironment("/", ds.ConfigRoot(), workspace, convertNetconf, sshKeyName) if len(userdataBytes) > 0 { if err := processUserdata(string(userdataBytes), env); err != nil { fmt.Printf("Failed resolving user-data: %v\n", err) diff --git a/datasource/configdrive.go b/datasource/configdrive.go index 7459471..7a6f6de 100644 --- a/datasource/configdrive.go +++ b/datasource/configdrive.go @@ -7,21 +7,29 @@ import ( ) type configDrive struct { - path string + root string } -func NewConfigDrive(path string) *configDrive { - return &configDrive{path} +func NewConfigDrive(root string) *configDrive { + return &configDrive{path.Join(root, "openstack")} +} + +func (self *configDrive) ConfigRoot() string { + return self.root } func (self *configDrive) Fetch() ([]byte, error) { - data, err := ioutil.ReadFile(path.Join(self.path, "openstack", "latest", "user_data")) - if os.IsNotExist(err) { - err = nil - } - return data, err + return self.readFile("user_data") } func (self *configDrive) Type() string { return "cloud-drive" } + +func (self *configDrive) readFile(filename string) ([]byte, error) { + data, err := ioutil.ReadFile(path.Join(self.root, "latest", filename)) + if os.IsNotExist(err) { + err = nil + } + return data, err +} diff --git a/datasource/datasource.go b/datasource/datasource.go index e3d2834..38d65fc 100644 --- a/datasource/datasource.go +++ b/datasource/datasource.go @@ -1,6 +1,7 @@ package datasource type Datasource interface { + ConfigRoot() string Fetch() ([]byte, error) Type() string } diff --git a/datasource/file.go b/datasource/file.go index c5cb1ae..24b57ef 100644 --- a/datasource/file.go +++ b/datasource/file.go @@ -12,6 +12,10 @@ func NewLocalFile(path string) *localFile { return &localFile{path} } +func (self *localFile) ConfigRoot() string { + return "" +} + func (self *localFile) Fetch() ([]byte, error) { return ioutil.ReadFile(self.path) } diff --git a/datasource/metadata_service.go b/datasource/metadata_service.go index 4ed036a..72b2665 100644 --- a/datasource/metadata_service.go +++ b/datasource/metadata_service.go @@ -10,6 +10,10 @@ func NewMetadataService(url string) *metadataService { return &metadataService{url} } +func (self *metadataService) ConfigRoot() string { + return "" +} + func (ms *metadataService) Fetch() ([]byte, error) { client := pkg.NewHttpClient() return client.Get(ms.url) diff --git a/datasource/proc_cmdline.go b/datasource/proc_cmdline.go index c389da3..3f861d0 100644 --- a/datasource/proc_cmdline.go +++ b/datasource/proc_cmdline.go @@ -14,7 +14,7 @@ const ( ProcCmdlineCloudConfigFlag = "cloud-config-url" ) -type procCmdline struct{ +type procCmdline struct { Location string } @@ -22,6 +22,10 @@ func NewProcCmdline() *procCmdline { return &procCmdline{Location: ProcCmdlineLocation} } +func (self *procCmdline) ConfigRoot() string { + return "" +} + func (self *procCmdline) Fetch() ([]byte, error) { contents, err := ioutil.ReadFile(self.Location) if err != nil { diff --git a/initialize/env.go b/initialize/env.go index b0ab3c1..948e51d 100644 --- a/initialize/env.go +++ b/initialize/env.go @@ -10,17 +10,19 @@ const DefaultSSHKeyName = "coreos-cloudinit" type Environment struct { root string + configRoot string workspace string + netconfType string sshKeyName string substitutions map[string]string } -func NewEnvironment(root, workspace, sshKeyName string) *Environment { +func NewEnvironment(root, configRoot, workspace, netconfType, sshKeyName string) *Environment { substitutions := map[string]string{ "$public_ipv4": os.Getenv("COREOS_PUBLIC_IPV4"), "$private_ipv4": os.Getenv("COREOS_PRIVATE_IPV4"), } - return &Environment{root, workspace, sshKeyName, substitutions} + return &Environment{root, configRoot, workspace, netconfType, sshKeyName, substitutions} } func (self *Environment) Workspace() string { @@ -31,6 +33,14 @@ func (self *Environment) Root() string { return self.root } +func (self *Environment) ConfigRoot() string { + return self.configRoot +} + +func (self *Environment) NetconfType() string { + return self.netconfType +} + func (self *Environment) SSHKeyName() string { return self.sshKeyName } diff --git a/initialize/env_test.go b/initialize/env_test.go index f10a054..33ff933 100644 --- a/initialize/env_test.go +++ b/initialize/env_test.go @@ -8,7 +8,7 @@ import ( func TestEnvironmentApply(t *testing.T) { os.Setenv("COREOS_PUBLIC_IPV4", "192.0.2.3") os.Setenv("COREOS_PRIVATE_IPV4", "192.0.2.203") - env := NewEnvironment("./", "./", "") + env := NewEnvironment("./", "./", "./", "", "") input := `[Service] ExecStart=/usr/bin/echo "$public_ipv4" ExecStop=/usr/bin/echo $private_ipv4 From 840c208b605682627149b123cf29501f87d414e7 Mon Sep 17 00:00:00 2001 From: Alex Crawford Date: Wed, 18 Jun 2014 11:58:18 -0700 Subject: [PATCH 3/4] feat(metadata): Distinguish between userdata and metadata for datasources --- coreos-cloudinit.go | 2 +- datasource/configdrive.go | 6 +++++- datasource/datasource.go | 3 ++- datasource/file.go | 6 +++++- datasource/metadata_service.go | 6 +++++- datasource/proc_cmdline.go | 6 +++++- datasource/proc_cmdline_test.go | 2 +- initialize/config.go | 9 +++++---- initialize/meta_data.go | 26 ++++++++++++++++++++++++++ 9 files changed, 55 insertions(+), 11 deletions(-) create mode 100644 initialize/meta_data.go diff --git a/coreos-cloudinit.go b/coreos-cloudinit.go index 99ceaac..50ee856 100644 --- a/coreos-cloudinit.go +++ b/coreos-cloudinit.go @@ -79,7 +79,7 @@ func main() { } fmt.Printf("Fetching user-data from datasource of type %q\n", ds.Type()) - userdataBytes, err := ds.Fetch() + userdataBytes, err := ds.FetchUserdata() if err != nil { fmt.Printf("Failed fetching user-data from datasource: %v\n", err) if ignoreFailure { diff --git a/datasource/configdrive.go b/datasource/configdrive.go index 7a6f6de..d995fb9 100644 --- a/datasource/configdrive.go +++ b/datasource/configdrive.go @@ -18,7 +18,11 @@ func (self *configDrive) ConfigRoot() string { return self.root } -func (self *configDrive) Fetch() ([]byte, error) { +func (self *configDrive) FetchMetadata() ([]byte, error) { + return self.readFile("meta_data.json") +} + +func (self *configDrive) FetchUserdata() ([]byte, error) { return self.readFile("user_data") } diff --git a/datasource/datasource.go b/datasource/datasource.go index 38d65fc..6c6aec4 100644 --- a/datasource/datasource.go +++ b/datasource/datasource.go @@ -2,6 +2,7 @@ package datasource type Datasource interface { ConfigRoot() string - Fetch() ([]byte, error) + FetchMetadata() ([]byte, error) + FetchUserdata() ([]byte, error) Type() string } diff --git a/datasource/file.go b/datasource/file.go index 24b57ef..33feb84 100644 --- a/datasource/file.go +++ b/datasource/file.go @@ -16,7 +16,11 @@ func (self *localFile) ConfigRoot() string { return "" } -func (self *localFile) Fetch() ([]byte, error) { +func (self *localFile) FetchMetadata() ([]byte, error) { + return []byte{}, nil +} + +func (self *localFile) FetchUserdata() ([]byte, error) { return ioutil.ReadFile(self.path) } diff --git a/datasource/metadata_service.go b/datasource/metadata_service.go index 72b2665..88b7d22 100644 --- a/datasource/metadata_service.go +++ b/datasource/metadata_service.go @@ -14,7 +14,11 @@ func (self *metadataService) ConfigRoot() string { return "" } -func (ms *metadataService) Fetch() ([]byte, error) { +func (self *metadataService) FetchMetadata() ([]byte, error) { + return []byte{}, nil +} + +func (ms *metadataService) FetchUserdata() ([]byte, error) { client := pkg.NewHttpClient() return client.Get(ms.url) } diff --git a/datasource/proc_cmdline.go b/datasource/proc_cmdline.go index 3f861d0..eb4d2d4 100644 --- a/datasource/proc_cmdline.go +++ b/datasource/proc_cmdline.go @@ -26,7 +26,11 @@ func (self *procCmdline) ConfigRoot() string { return "" } -func (self *procCmdline) Fetch() ([]byte, error) { +func (self *procCmdline) FetchMetadata() ([]byte, error) { + return []byte{}, nil +} + +func (self *procCmdline) FetchUserdata() ([]byte, error) { contents, err := ioutil.ReadFile(self.Location) if err != nil { return nil, err diff --git a/datasource/proc_cmdline_test.go b/datasource/proc_cmdline_test.go index bee6e64..65115c9 100644 --- a/datasource/proc_cmdline_test.go +++ b/datasource/proc_cmdline_test.go @@ -77,7 +77,7 @@ func TestProcCmdlineAndFetchConfig(t *testing.T) { p := NewProcCmdline() p.Location = file.Name() - cfg, err := p.Fetch() + cfg, err := p.FetchUserdata() if err != nil { t.Errorf("Test produced error: %v", err) } diff --git a/initialize/config.go b/initialize/config.go index 3f59e28..5d4e27b 100644 --- a/initialize/config.go +++ b/initialize/config.go @@ -34,10 +34,11 @@ type CloudConfig struct { Update UpdateConfig Units []system.Unit } - WriteFiles []system.File `yaml:"write_files"` - Hostname string - Users []system.User - ManageEtcHosts EtcHosts `yaml:"manage_etc_hosts"` + WriteFiles []system.File `yaml:"write_files"` + Hostname string + Users []system.User + ManageEtcHosts EtcHosts `yaml:"manage_etc_hosts"` + NetworkConfigPath string } type warner func(format string, v ...interface{}) diff --git a/initialize/meta_data.go b/initialize/meta_data.go new file mode 100644 index 0000000..d83b8b5 --- /dev/null +++ b/initialize/meta_data.go @@ -0,0 +1,26 @@ +package initialize + +import ( + "encoding/json" +) + +func ParseMetaData(contents string) (cfg CloudConfig, err error) { + var metadata struct { + SSHAuthorizedKeyMap map[string]string `json:"public_keys"` + Hostname string `json:"hostname"` + NetworkConfig struct { + ContentPath string `json:"content_path"` + } `json:"network_config"` + } + if err = json.Unmarshal([]byte(contents), &metadata); err != nil { + return + } + + cfg.SSHAuthorizedKeys = make([]string, 0, len(metadata.SSHAuthorizedKeyMap)) + for _, key := range metadata.SSHAuthorizedKeyMap { + cfg.SSHAuthorizedKeys = append(cfg.SSHAuthorizedKeys, key) + } + cfg.Hostname = metadata.Hostname + cfg.NetworkConfigPath = metadata.NetworkConfig.ContentPath + return +} From e6cf83a2e536ea2a3ee1455d502d53d2ab89039b Mon Sep 17 00:00:00 2001 From: Alex Crawford Date: Wed, 18 Jun 2014 12:08:10 -0700 Subject: [PATCH 4/4] refactor(netconf): Move netconf processing and handle metadata --- coreos-cloudinit.go | 69 +++++++++++++++----------------------------- initialize/config.go | 29 +++++++++++++++++++ 2 files changed, 53 insertions(+), 45 deletions(-) diff --git a/coreos-cloudinit.go b/coreos-cloudinit.go index 50ee856..fb0b63f 100644 --- a/coreos-cloudinit.go +++ b/coreos-cloudinit.go @@ -1,16 +1,12 @@ package main import ( - "encoding/json" "flag" "fmt" - "io/ioutil" "os" - "path" "github.com/coreos/coreos-cloudinit/datasource" "github.com/coreos/coreos-cloudinit/initialize" - "github.com/coreos/coreos-cloudinit/network" "github.com/coreos/coreos-cloudinit/system" ) @@ -89,25 +85,38 @@ func main() { } } + fmt.Printf("Fetching meta-data from datasource of type %q\n", ds.Type()) + metadataBytes, err := ds.FetchMetadata() + if err != nil { + fmt.Printf("Failed fetching meta-data from datasource: %v\n", err) + if ignoreFailure { + os.Exit(0) + } else { + os.Exit(1) + } + } + env := initialize.NewEnvironment("/", ds.ConfigRoot(), workspace, convertNetconf, sshKeyName) if len(userdataBytes) > 0 { if err := processUserdata(string(userdataBytes), env); err != nil { - fmt.Printf("Failed resolving user-data: %v\n", err) + fmt.Printf("Failed to process user-data: %v\n", err) if !ignoreFailure { os.Exit(1) } } } else { - fmt.Println("No user data to handle.") + fmt.Println("No user-data to handle.") } - if convertNetconf != "" { - if err := processNetconf(convertNetconf, configdrive); err != nil { - fmt.Printf("Failed to process network config: %v\n", err) + if len(metadataBytes) > 0 { + if err := processMetadata(string(metadataBytes), env); err != nil { + fmt.Printf("Failed to process meta-data: %v\n", err) if !ignoreFailure { os.Exit(1) } } + } else { + fmt.Println("No meta-data to handle.") } } @@ -142,47 +151,17 @@ func processUserdata(userdata string, env *initialize.Environment) error { return err } -func processNetconf(convertNetconf, configdrive string) error { - openstackRoot := path.Join(configdrive, "openstack") - metadataFilename := path.Join(openstackRoot, "latest", "meta_data.json") - metadataBytes, err := ioutil.ReadFile(metadataFilename) +func processMetadata(metadata string, env *initialize.Environment) error { + parsed, err := initialize.ParseMetaData(metadata) if err != nil { + fmt.Printf("Failed parsing meta-data: %v\n", err) return err } - - var metadata struct { - NetworkConfig struct { - ContentPath string `json:"content_path"` - } `json:"network_config"` - } - if err := json.Unmarshal(metadataBytes, &metadata); err != nil { - return err - } - configPath := metadata.NetworkConfig.ContentPath - if configPath == "" { - fmt.Printf("No network config specified in %q.\n", metadataFilename) - return nil - } - - netconfBytes, err := ioutil.ReadFile(path.Join(openstackRoot, configPath)) + err = initialize.PrepWorkspace(env.Workspace()) if err != nil { + fmt.Printf("Failed preparing workspace: %v\n", err) return err } - var interfaces []network.InterfaceGenerator - switch convertNetconf { - case "debian": - interfaces, err = network.ProcessDebianNetconf(string(netconfBytes)) - default: - return fmt.Errorf("Unsupported network config format %q", convertNetconf) - } - - if err != nil { - return err - } - - if err := system.WriteNetworkdConfigs(interfaces); err != nil { - return err - } - return system.RestartNetwork(interfaces) + return initialize.Apply(parsed, env) } diff --git a/initialize/config.go b/initialize/config.go index 5d4e27b..4a73997 100644 --- a/initialize/config.go +++ b/initialize/config.go @@ -3,10 +3,13 @@ package initialize import ( "errors" "fmt" + "io/ioutil" "log" + "path" "github.com/coreos/coreos-cloudinit/third_party/launchpad.net/goyaml" + "github.com/coreos/coreos-cloudinit/network" "github.com/coreos/coreos-cloudinit/system" ) @@ -228,6 +231,32 @@ func Apply(cfg CloudConfig, env *Environment) error { log.Printf("Wrote file %s to filesystem", path) } + if env.NetconfType() != "" { + netconfBytes, err := ioutil.ReadFile(path.Join(env.ConfigRoot(), cfg.NetworkConfigPath)) + if err != nil { + return err + } + + var interfaces []network.InterfaceGenerator + switch env.NetconfType() { + case "debian": + interfaces, err = network.ProcessDebianNetconf(string(netconfBytes)) + default: + return fmt.Errorf("Unsupported network config format %q", env.NetconfType()) + } + + if err != nil { + return err + } + + if err := system.WriteNetworkdConfigs(interfaces); err != nil { + return err + } + if err := system.RestartNetwork(interfaces); err != nil { + return err + } + } + commands := make(map[string]string, 0) reload := false for _, unit := range cfg.Coreos.Units {