diff --git a/coreos-cloudinit.go b/coreos-cloudinit.go index 391ed2f..30a250b 100644 --- a/coreos-cloudinit.go +++ b/coreos-cloudinit.go @@ -18,6 +18,7 @@ var ( sources struct { file string configDrive string + metadataService string url string procCmdLine bool } @@ -30,7 +31,8 @@ func init() { flag.BoolVar(&printVersion, "version", false, "Print the version and exit") flag.BoolVar(&ignoreFailure, "ignore-failure", false, "Exits with 0 status in the event of malformed input from user-data") flag.StringVar(&sources.file, "from-file", "", "Read user-data from provided file") - flag.StringVar(&sources.configDrive, "from-configdrive", "", "Read user-data from provided cloud-drive directory") + flag.StringVar(&sources.configDrive, "from-configdrive", "", "Read data from provided cloud-drive directory") + flag.StringVar(&sources.metadataService, "from-metadata-service", "", "Download data from provided url") flag.StringVar(&sources.url, "from-url", "", "Download user-data from provided url") flag.BoolVar(&sources.procCmdLine, "from-proc-cmdline", false, fmt.Sprintf("Parse %s for '%s=', using the cloud-config served by an HTTP GET to ", datasource.ProcCmdlineLocation, datasource.ProcCmdlineCloudConfigFlag)) flag.StringVar(&convertNetconf, "convert-netconf", "", "Read the network config provided in cloud-drive and translate it from the specified format into networkd unit files (requires the -from-configdrive flag)") @@ -53,8 +55,8 @@ func main() { os.Exit(0) } - if convertNetconf != "" && sources.configDrive == "" { - fmt.Println("-convert-netconf flag requires -from-configdrive") + if convertNetconf != "" && sources.configDrive == "" && sources.metadataService == "" { + fmt.Println("-convert-netconf flag requires -from-configdrive or -from-metadata-service") os.Exit(1) } @@ -68,7 +70,7 @@ func main() { ds := getDatasource() if ds == nil { - fmt.Println("Provide exactly one of --from-file, --from-configdrive, --from-url or --from-proc-cmdline") + fmt.Println("Provide exactly one of --from-file, --from-configdrive, --from-metadata-service, --from-url or --from-proc-cmdline") os.Exit(1) } @@ -123,6 +125,10 @@ func getDatasource() datasource.Datasource { ds = datasource.NewConfigDrive(sources.configDrive) n++ } + if sources.metadataService != "" { + ds = datasource.NewMetadataService(sources.metadataService) + n++ + } if sources.procCmdLine { ds = datasource.NewProcCmdline() n++ diff --git a/datasource/configdrive.go b/datasource/configdrive.go index d995fb9..ca618f3 100644 --- a/datasource/configdrive.go +++ b/datasource/configdrive.go @@ -14,24 +14,24 @@ func NewConfigDrive(root string) *configDrive { return &configDrive{path.Join(root, "openstack")} } -func (self *configDrive) ConfigRoot() string { - return self.root +func (cd *configDrive) ConfigRoot() string { + return cd.root } -func (self *configDrive) FetchMetadata() ([]byte, error) { - return self.readFile("meta_data.json") +func (cd *configDrive) FetchMetadata() ([]byte, error) { + return cd.readFile("meta_data.json") } -func (self *configDrive) FetchUserdata() ([]byte, error) { - return self.readFile("user_data") +func (cd *configDrive) FetchUserdata() ([]byte, error) { + return cd.readFile("user_data") } -func (self *configDrive) Type() string { +func (cd *configDrive) Type() string { return "cloud-drive" } -func (self *configDrive) readFile(filename string) ([]byte, error) { - data, err := ioutil.ReadFile(path.Join(self.root, "latest", filename)) +func (cd *configDrive) readFile(filename string) ([]byte, error) { + data, err := ioutil.ReadFile(path.Join(cd.root, "latest", filename)) if os.IsNotExist(err) { err = nil } diff --git a/datasource/file.go b/datasource/file.go index 33feb84..0798b7c 100644 --- a/datasource/file.go +++ b/datasource/file.go @@ -12,18 +12,18 @@ func NewLocalFile(path string) *localFile { return &localFile{path} } -func (self *localFile) ConfigRoot() string { +func (f *localFile) ConfigRoot() string { return "" } -func (self *localFile) FetchMetadata() ([]byte, error) { +func (f *localFile) FetchMetadata() ([]byte, error) { return []byte{}, nil } -func (self *localFile) FetchUserdata() ([]byte, error) { - return ioutil.ReadFile(self.path) +func (f *localFile) FetchUserdata() ([]byte, error) { + return ioutil.ReadFile(f.path) } -func (self *localFile) Type() string { +func (f *localFile) Type() string { return "local-file" } diff --git a/datasource/metadata_service.go b/datasource/metadata_service.go index 88b7d22..79ee559 100644 --- a/datasource/metadata_service.go +++ b/datasource/metadata_service.go @@ -1,28 +1,97 @@ package datasource -import "github.com/coreos/coreos-cloudinit/pkg" +import ( + "bufio" + "bytes" + "encoding/json" + "strings" + + "github.com/coreos/coreos-cloudinit/pkg" +) type metadataService struct { url string } -func NewMetadataService(url string) *metadataService { - return &metadataService{url} +type getter interface { + Get(string) ([]byte, error) } -func (self *metadataService) ConfigRoot() string { +func NewMetadataService(url string) *metadataService { + return &metadataService{strings.TrimSuffix(url, "/")} +} + +func (ms *metadataService) ConfigRoot() string { return "" } -func (self *metadataService) FetchMetadata() ([]byte, error) { - return []byte{}, nil +func (ms *metadataService) FetchMetadata() ([]byte, error) { + client := pkg.NewHttpClient() + return fetchMetadata(client, ms.url) } func (ms *metadataService) FetchUserdata() ([]byte, error) { client := pkg.NewHttpClient() - return client.Get(ms.url) + return client.Get(ms.url + "/latest/user-data") } func (ms *metadataService) Type() string { return "metadata-service" } + +func fetchMetadata(client getter, url string) ([]byte, error) { + if metadata, err := client.Get(url + "/latest/meta-data.json"); err == nil { + return metadata, nil + } else if _, ok := err.(pkg.ErrTimeout); ok { + return nil, err + } + + attrs, err := fetchChildAttributes(client, url+"/latest/meta-data/") + if err != nil { + return nil, err + } + return json.Marshal(attrs) +} + +func fetchAttributes(client getter, url string) ([]string, error) { + resp, err := client.Get(url) + if err != nil { + return nil, err + } + scanner := bufio.NewScanner(bytes.NewBuffer(resp)) + data := make([]string, 0) + for scanner.Scan() { + data = append(data, strings.Split(scanner.Text(), "=")[0]) + } + return data, scanner.Err() +} + +func fetchAttribute(client getter, url string) (interface{}, error) { + if attrs, err := fetchAttributes(client, url); err == nil { + return attrs[0], nil + } else { + return "", err + } +} + +func fetchChildAttributes(client getter, url string) (interface{}, error) { + attrs := make(map[string]interface{}) + attrList, err := fetchAttributes(client, url) + if err != nil { + return nil, err + } + for _, attr := range attrList { + var fetchFunc func(getter, string) (interface{}, error) + if strings.HasSuffix(attr, "/") { + fetchFunc = fetchChildAttributes + } else { + fetchFunc = fetchAttribute + } + if value, err := fetchFunc(client, url+attr); err == nil { + attrs[strings.TrimSuffix(attr, "/")] = value + } else { + return nil, err + } + } + return attrs, nil +} diff --git a/datasource/metadata_service_test.go b/datasource/metadata_service_test.go new file mode 100644 index 0000000..0089533 --- /dev/null +++ b/datasource/metadata_service_test.go @@ -0,0 +1,168 @@ +package datasource + +import ( + "bytes" + "fmt" + "reflect" + "testing" + + "github.com/coreos/coreos-cloudinit/pkg" +) + +type TestHttpClient struct { + metadata map[string]string + err error +} + +func (t *TestHttpClient) Get(url string) ([]byte, error) { + if t.err != nil { + return nil, t.err + } + if val, ok := t.metadata[url]; ok { + return []byte(val), nil + } else { + return nil, pkg.ErrNotFound{fmt.Errorf("not found: %q", url)} + } +} + +func TestFetchAttributes(t *testing.T) { + for _, s := range []struct { + metadata map[string]string + err error + tests []struct { + path string + val []string + } + }{ + { + metadata: map[string]string{ + "/": "a\nb\nc/", + "/c/": "d\ne/", + "/c/e/": "f", + "/a": "1", + "/b": "2", + "/c/d": "3", + "/c/e/f": "4", + }, + tests: []struct { + path string + val []string + }{ + {"/", []string{"a", "b", "c/"}}, + {"/b", []string{"2"}}, + {"/c/d", []string{"3"}}, + {"/c/e/", []string{"f"}}, + }, + }, + { + err: pkg.ErrNotFound{fmt.Errorf("test error")}, + tests: []struct { + path string + val []string + }{ + {"", nil}, + }, + }, + } { + client := &TestHttpClient{s.metadata, s.err} + for _, tt := range s.tests { + attrs, err := fetchAttributes(client, tt.path) + if err != s.err { + t.Fatalf("bad error for %q (%q): want %q, got %q", tt.path, s.metadata, s.err, err) + } + if !reflect.DeepEqual(attrs, tt.val) { + t.Fatalf("bad fetch for %q (%q): want %q, got %q", tt.path, s.metadata, tt.val, attrs) + } + } + } +} + +func TestFetchAttribute(t *testing.T) { + for _, s := range []struct { + metadata map[string]string + err error + tests []struct { + path string + val string + } + }{ + { + metadata: map[string]string{ + "/": "a\nb\nc/", + "/c/": "d\ne/", + "/c/e/": "f", + "/a": "1", + "/b": "2", + "/c/d": "3", + "/c/e/f": "4", + }, + tests: []struct { + path string + val string + }{ + {"/a", "1"}, + {"/b", "2"}, + {"/c/d", "3"}, + {"/c/e/f", "4"}, + }, + }, + { + err: pkg.ErrNotFound{fmt.Errorf("test error")}, + tests: []struct { + path string + val string + }{ + {"", ""}, + }, + }, + } { + client := &TestHttpClient{s.metadata, s.err} + for _, tt := range s.tests { + attr, err := fetchAttribute(client, tt.path) + if err != s.err { + t.Fatalf("bad error for %q (%q): want %q, got %q", tt.path, s.metadata, s.err, err) + } + if attr != tt.val { + t.Fatalf("bad fetch for %q (%q): want %q, got %q", tt.path, s.metadata, tt.val, attr) + } + } + } +} + +func TestFetchMetadata(t *testing.T) { + for _, tt := range []struct { + metadata map[string]string + err error + expect []byte + }{ + { + metadata: map[string]string{ + "/latest/meta-data/": "a\nb\nc/", + "/latest/meta-data/c/": "d\ne/", + "/latest/meta-data/c/e/": "f", + "/latest/meta-data/a": "1", + "/latest/meta-data/b": "2", + "/latest/meta-data/c/d": "3", + "/latest/meta-data/c/e/f": "4", + }, + expect: []byte(`{"a":"1","b":"2","c":{"d":"3","e":{"f":"4"}}}`), + }, + { + metadata: map[string]string{ + "/latest/meta-data.json": "test", + }, + expect: []byte("test"), + }, + {err: pkg.ErrTimeout{fmt.Errorf("test error")}}, + {err: pkg.ErrNotFound{fmt.Errorf("test error")}}, + } { + client := &TestHttpClient{tt.metadata, tt.err} + metadata, err := fetchMetadata(client, "") + if err != tt.err { + t.Fatalf("bad error (%q): want %q, got %q", tt.metadata, tt.err, err) + } + if !bytes.Equal(metadata, tt.expect) { + t.Fatalf("bad fetch (%q): want %q, got %q", tt.metadata, tt.expect, metadata) + } + } +} diff --git a/datasource/proc_cmdline.go b/datasource/proc_cmdline.go index eb4d2d4..ce181ab 100644 --- a/datasource/proc_cmdline.go +++ b/datasource/proc_cmdline.go @@ -22,16 +22,16 @@ func NewProcCmdline() *procCmdline { return &procCmdline{Location: ProcCmdlineLocation} } -func (self *procCmdline) ConfigRoot() string { +func (c *procCmdline) ConfigRoot() string { return "" } -func (self *procCmdline) FetchMetadata() ([]byte, error) { +func (c *procCmdline) FetchMetadata() ([]byte, error) { return []byte{}, nil } -func (self *procCmdline) FetchUserdata() ([]byte, error) { - contents, err := ioutil.ReadFile(self.Location) +func (c *procCmdline) FetchUserdata() ([]byte, error) { + contents, err := ioutil.ReadFile(c.Location) if err != nil { return nil, err } @@ -51,7 +51,7 @@ func (self *procCmdline) FetchUserdata() ([]byte, error) { return cfg, nil } -func (self *procCmdline) Type() string { +func (c *procCmdline) Type() string { return "proc-cmdline" } diff --git a/datasource/url.go b/datasource/url.go new file mode 100644 index 0000000..9fcc788 --- /dev/null +++ b/datasource/url.go @@ -0,0 +1,28 @@ +package datasource + +import "github.com/coreos/coreos-cloudinit/pkg" + +type remoteFile struct { + url string +} + +func NewRemoteFile(url string) *remoteFile { + return &remoteFile{url} +} + +func (f *remoteFile) ConfigRoot() string { + return "" +} + +func (f *remoteFile) FetchMetadata() ([]byte, error) { + return []byte{}, nil +} + +func (f *remoteFile) FetchUserdata() ([]byte, error) { + client := pkg.NewHttpClient() + return client.Get(f.url) +} + +func (f *remoteFile) Type() string { + return "url" +} diff --git a/initialize/env.go b/initialize/env.go index 948e51d..781a77c 100644 --- a/initialize/env.go +++ b/initialize/env.go @@ -25,32 +25,32 @@ func NewEnvironment(root, configRoot, workspace, netconfType, sshKeyName string) return &Environment{root, configRoot, workspace, netconfType, sshKeyName, substitutions} } -func (self *Environment) Workspace() string { - return path.Join(self.root, self.workspace) +func (e *Environment) Workspace() string { + return path.Join(e.root, e.workspace) } -func (self *Environment) Root() string { - return self.root +func (e *Environment) Root() string { + return e.root } -func (self *Environment) ConfigRoot() string { - return self.configRoot +func (e *Environment) ConfigRoot() string { + return e.configRoot } -func (self *Environment) NetconfType() string { - return self.netconfType +func (e *Environment) NetconfType() string { + return e.netconfType } -func (self *Environment) SSHKeyName() string { - return self.sshKeyName +func (e *Environment) SSHKeyName() string { + return e.sshKeyName } -func (self *Environment) SetSSHKeyName(name string) { - self.sshKeyName = name +func (e *Environment) SetSSHKeyName(name string) { + e.sshKeyName = name } -func (self *Environment) Apply(data string) string { - for key, val := range self.substitutions { +func (e *Environment) Apply(data string) string { + for key, val := range e.substitutions { data = strings.Replace(data, key, val, -1) } return data diff --git a/pkg/http_client.go b/pkg/http_client.go index 59e055f..5152fef 100644 --- a/pkg/http_client.go +++ b/pkg/http_client.go @@ -18,6 +18,20 @@ const ( HTTP_4xx = 4 ) +type Err error + +type ErrTimeout struct{ + Err +} + +type ErrNotFound struct{ + Err +} + +type ErrInvalid struct{ + Err +} + type HttpClient struct { // Maximum exp backoff duration. Defaults to 5 seconds MaxBackoff time.Duration @@ -53,18 +67,18 @@ func expBackoff(interval, max time.Duration) time.Duration { // Fetches a given URL with support for exponential backoff and maximum retries func (h *HttpClient) Get(rawurl string) ([]byte, error) { if rawurl == "" { - return nil, errors.New("URL is empty. Skipping.") + return nil, ErrInvalid{errors.New("URL is empty. Skipping.")} } url, err := neturl.Parse(rawurl) if err != nil { - return nil, err + return nil, ErrInvalid{err} } // Unfortunately, url.Parse is too generic to throw errors if a URL does not // have a valid HTTP scheme. So, we have to do this extra validation if !strings.HasPrefix(url.Scheme, "http") { - return nil, fmt.Errorf("URL %s does not have a valid HTTP scheme. Skipping.", rawurl) + return nil, ErrInvalid{fmt.Errorf("URL %s does not have a valid HTTP scheme. Skipping.", rawurl)} } dataURL := url.String() @@ -106,7 +120,7 @@ func (h *HttpClient) Get(rawurl string) ([]byte, error) { } if status == HTTP_4xx { - return nil, fmt.Errorf("Not found. HTTP status code: %d", resp.StatusCode) + return nil, ErrNotFound{fmt.Errorf("Not found. HTTP status code: %d", resp.StatusCode)} } log.Printf("Server error. HTTP status code: %d", resp.StatusCode) @@ -119,5 +133,5 @@ func (h *HttpClient) Get(rawurl string) ([]byte, error) { time.Sleep(duration) } - return nil, fmt.Errorf("Unable to fetch data. Maximum retries reached: %d", h.MaxRetries) + return nil, ErrTimeout{fmt.Errorf("Unable to fetch data. Maximum retries reached: %d", h.MaxRetries)} }