Merge pull request #194 from crawford/metadata
datasource: Refactoring datasources
This commit is contained in:
Alex Crawford
@@ -8,6 +8,11 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/coreos/coreos-cloudinit/datasource"
|
||||
"github.com/coreos/coreos-cloudinit/datasource/configdrive"
|
||||
"github.com/coreos/coreos-cloudinit/datasource/file"
|
||||
"github.com/coreos/coreos-cloudinit/datasource/metadata/ec2"
|
||||
"github.com/coreos/coreos-cloudinit/datasource/proc_cmdline"
|
||||
"github.com/coreos/coreos-cloudinit/datasource/url"
|
||||
"github.com/coreos/coreos-cloudinit/initialize"
|
||||
"github.com/coreos/coreos-cloudinit/pkg"
|
||||
"github.com/coreos/coreos-cloudinit/system"
|
||||
@@ -24,11 +29,12 @@ var (
|
||||
printVersion bool
|
||||
ignoreFailure bool
|
||||
sources struct {
|
||||
file string
|
||||
configDrive string
|
||||
metadataService bool
|
||||
url string
|
||||
procCmdLine bool
|
||||
file string
|
||||
configDrive string
|
||||
metadataService bool
|
||||
ec2MetadataService string
|
||||
url string
|
||||
procCmdLine bool
|
||||
}
|
||||
convertNetconf string
|
||||
workspace string
|
||||
@@ -40,9 +46,10 @@ func init() {
|
||||
flag.BoolVar(&ignoreFailure, "ignore-failure", false, "Exits with 0 status in the event of malformed input from user-data")
|
||||
flag.StringVar(&sources.file, "from-file", "", "Read user-data from provided file")
|
||||
flag.StringVar(&sources.configDrive, "from-configdrive", "", "Read data from provided cloud-drive directory")
|
||||
flag.BoolVar(&sources.metadataService, "from-metadata-service", false, "Download data from metadata service")
|
||||
flag.BoolVar(&sources.metadataService, "from-metadata-service", false, "[DEPRECATED - Use -from-ec2-metadata] Download data from metadata service")
|
||||
flag.StringVar(&sources.ec2MetadataService, "from-ec2-metadata", "", "Download data from the provided metadata service")
|
||||
flag.StringVar(&sources.url, "from-url", "", "Download user-data from provided url")
|
||||
flag.BoolVar(&sources.procCmdLine, "from-proc-cmdline", false, fmt.Sprintf("Parse %s for '%s=<url>', using the cloud-config served by an HTTP GET to <url>", datasource.ProcCmdlineLocation, datasource.ProcCmdlineCloudConfigFlag))
|
||||
flag.BoolVar(&sources.procCmdLine, "from-proc-cmdline", false, fmt.Sprintf("Parse %s for '%s=<url>', using the cloud-config served by an HTTP GET to <url>", proc_cmdline.ProcCmdlineLocation, proc_cmdline.ProcCmdlineCloudConfigFlag))
|
||||
flag.StringVar(&convertNetconf, "convert-netconf", "", "Read the network config provided in cloud-drive and translate it from the specified format into networkd unit files (requires the -from-configdrive flag)")
|
||||
flag.StringVar(&workspace, "workspace", "/var/lib/coreos-cloudinit", "Base directory coreos-cloudinit should use to store data")
|
||||
flag.StringVar(&sshKeyName, "ssh-key-name", initialize.DefaultSSHKeyName, "Add SSH keys to the system with the given name")
|
||||
@@ -78,7 +85,7 @@ func main() {
|
||||
|
||||
dss := getDatasources()
|
||||
if len(dss) == 0 {
|
||||
fmt.Println("Provide at least one of --from-file, --from-configdrive, --from-metadata-service, --from-url or --from-proc-cmdline")
|
||||
fmt.Println("Provide at least one of --from-file, --from-configdrive, --from-ec2-metadata, --from-url or --from-proc-cmdline")
|
||||
os.Exit(1)
|
||||
}
|
||||
|
||||
@@ -172,7 +179,7 @@ func main() {
|
||||
func mergeCloudConfig(mdcc, udcc initialize.CloudConfig) (cc initialize.CloudConfig) {
|
||||
if mdcc.Hostname != "" {
|
||||
if udcc.Hostname != "" {
|
||||
fmt.Printf("Warning: user-data hostname (%s) overrides metadata hostname (%s)", udcc.Hostname, mdcc.Hostname)
|
||||
fmt.Printf("Warning: user-data hostname (%s) overrides metadata hostname (%s)\n", udcc.Hostname, mdcc.Hostname)
|
||||
} else {
|
||||
udcc.Hostname = mdcc.Hostname
|
||||
}
|
||||
@@ -183,7 +190,7 @@ func mergeCloudConfig(mdcc, udcc initialize.CloudConfig) (cc initialize.CloudCon
|
||||
}
|
||||
if mdcc.NetworkConfigPath != "" {
|
||||
if udcc.NetworkConfigPath != "" {
|
||||
fmt.Printf("Warning: user-data NetworkConfigPath %s overrides metadata NetworkConfigPath %s", udcc.NetworkConfigPath, mdcc.NetworkConfigPath)
|
||||
fmt.Printf("Warning: user-data NetworkConfigPath %s overrides metadata NetworkConfigPath %s\n", udcc.NetworkConfigPath, mdcc.NetworkConfigPath)
|
||||
} else {
|
||||
udcc.NetworkConfigPath = mdcc.NetworkConfigPath
|
||||
}
|
||||
@@ -196,19 +203,22 @@ func mergeCloudConfig(mdcc, udcc initialize.CloudConfig) (cc initialize.CloudCon
|
||||
func getDatasources() []datasource.Datasource {
|
||||
dss := make([]datasource.Datasource, 0, 5)
|
||||
if sources.file != "" {
|
||||
dss = append(dss, datasource.NewLocalFile(sources.file))
|
||||
dss = append(dss, file.NewDatasource(sources.file))
|
||||
}
|
||||
if sources.url != "" {
|
||||
dss = append(dss, datasource.NewRemoteFile(sources.url))
|
||||
dss = append(dss, url.NewDatasource(sources.url))
|
||||
}
|
||||
if sources.configDrive != "" {
|
||||
dss = append(dss, datasource.NewConfigDrive(sources.configDrive))
|
||||
dss = append(dss, configdrive.NewDatasource(sources.configDrive))
|
||||
}
|
||||
if sources.metadataService {
|
||||
dss = append(dss, datasource.NewMetadataService())
|
||||
dss = append(dss, ec2.NewDatasource(ec2.DefaultAddress))
|
||||
}
|
||||
if sources.ec2MetadataService != "" {
|
||||
dss = append(dss, ec2.NewDatasource(sources.ec2MetadataService))
|
||||
}
|
||||
if sources.procCmdLine {
|
||||
dss = append(dss, datasource.NewProcCmdline())
|
||||
dss = append(dss, proc_cmdline.NewDatasource())
|
||||
}
|
||||
return dss
|
||||
}
|
||||
@@ -240,7 +250,7 @@ func selectDatasource(sources []datasource.Datasource) datasource.Datasource {
|
||||
select {
|
||||
case <-stop:
|
||||
return
|
||||
case <-time.Tick(duration):
|
||||
case <-time.After(duration):
|
||||
duration = pkg.ExpBackoff(duration, datasourceMaxInterval)
|
||||
}
|
||||
}
|
||||
@@ -257,7 +267,7 @@ func selectDatasource(sources []datasource.Datasource) datasource.Datasource {
|
||||
select {
|
||||
case s = <-ds:
|
||||
case <-done:
|
||||
case <-time.Tick(datasourceTimeout):
|
||||
case <-time.After(datasourceTimeout):
|
||||
}
|
||||
|
||||
close(stop)
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
package datasource
|
||||
package configdrive
|
||||
|
||||
import (
|
||||
"io/ioutil"
|
||||
@@ -6,13 +6,18 @@ import (
|
||||
"path"
|
||||
)
|
||||
|
||||
const (
|
||||
ec2ApiVersion = "2009-04-04"
|
||||
openstackApiVersion = "latest"
|
||||
)
|
||||
|
||||
type configDrive struct {
|
||||
root string
|
||||
readFile func(filename string) ([]byte, error)
|
||||
}
|
||||
|
||||
func NewConfigDrive(root string) *configDrive {
|
||||
return &configDrive{root, ioutil.ReadFile}
|
||||
func NewDatasource(root string) *configDrive {
|
||||
return &configDrive{path.Join(root, "openstack"), ioutil.ReadFile}
|
||||
}
|
||||
|
||||
func (cd *configDrive) IsAvailable() bool {
|
||||
@@ -48,11 +53,11 @@ func (cd *configDrive) Type() string {
|
||||
}
|
||||
|
||||
func (cd *configDrive) ec2Root() string {
|
||||
return path.Join(cd.root, "ec2", Ec2ApiVersion)
|
||||
return path.Join(cd.root, "ec2", ec2ApiVersion)
|
||||
}
|
||||
|
||||
func (cd *configDrive) openstackRoot() string {
|
||||
return path.Join(cd.root, "openstack", "latest")
|
||||
return path.Join(cd.root, "openstack", openstackApiVersion)
|
||||
}
|
||||
|
||||
func (cd *configDrive) tryReadFile(filename string) ([]byte, error) {
|
||||
@@ -1,4 +1,4 @@
|
||||
package datasource
|
||||
package configdrive
|
||||
|
||||
import (
|
||||
"os"
|
||||
@@ -1,4 +1,4 @@
|
||||
package datasource
|
||||
package file
|
||||
|
||||
import (
|
||||
"io/ioutil"
|
||||
@@ -9,7 +9,7 @@ type localFile struct {
|
||||
path string
|
||||
}
|
||||
|
||||
func NewLocalFile(path string) *localFile {
|
||||
func NewDatasource(path string) *localFile {
|
||||
return &localFile{path}
|
||||
}
|
||||
|
||||
141
datasource/metadata/ec2/metadata.go
Normal file
141
datasource/metadata/ec2/metadata.go
Normal file
@@ -0,0 +1,141 @@
|
||||
package ec2
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/coreos/coreos-cloudinit/pkg"
|
||||
)
|
||||
|
||||
const (
|
||||
DefaultAddress = "http://169.254.169.254/"
|
||||
apiVersion = "2009-04-04"
|
||||
userdataUrl = apiVersion + "/user-data"
|
||||
metadataUrl = apiVersion + "/meta-data"
|
||||
)
|
||||
|
||||
type metadataService struct {
|
||||
root string
|
||||
client pkg.Getter
|
||||
}
|
||||
|
||||
func NewDatasource(root string) *metadataService {
|
||||
if !strings.HasSuffix(root, "/") {
|
||||
root += "/"
|
||||
}
|
||||
return &metadataService{root, pkg.NewHttpClient()}
|
||||
}
|
||||
|
||||
func (ms metadataService) IsAvailable() bool {
|
||||
_, err := ms.client.Get(ms.root + apiVersion)
|
||||
return (err == nil)
|
||||
}
|
||||
|
||||
func (ms metadataService) AvailabilityChanges() bool {
|
||||
return true
|
||||
}
|
||||
|
||||
func (ms metadataService) ConfigRoot() string {
|
||||
return ms.root
|
||||
}
|
||||
|
||||
func (ms metadataService) FetchMetadata() ([]byte, error) {
|
||||
attrs := make(map[string]interface{})
|
||||
if keynames, err := fetchAttributes(ms.client, fmt.Sprintf("%s/public-keys", ms.metadataUrl())); err == nil {
|
||||
keyIDs := make(map[string]string)
|
||||
for _, keyname := range keynames {
|
||||
tokens := strings.SplitN(keyname, "=", 2)
|
||||
if len(tokens) != 2 {
|
||||
return nil, fmt.Errorf("malformed public key: %q", keyname)
|
||||
}
|
||||
keyIDs[tokens[1]] = tokens[0]
|
||||
}
|
||||
|
||||
keys := make(map[string]string)
|
||||
for name, id := range keyIDs {
|
||||
sshkey, err := fetchAttribute(ms.client, fmt.Sprintf("%s/public-keys/%s/openssh-key", ms.metadataUrl(), id))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
keys[name] = sshkey
|
||||
fmt.Printf("Found SSH key for %q\n", name)
|
||||
}
|
||||
attrs["public_keys"] = keys
|
||||
} else if _, ok := err.(pkg.ErrNotFound); !ok {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if hostname, err := fetchAttribute(ms.client, fmt.Sprintf("%s/hostname", ms.metadataUrl())); err == nil {
|
||||
attrs["hostname"] = hostname
|
||||
} else if _, ok := err.(pkg.ErrNotFound); !ok {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if localAddr, err := fetchAttribute(ms.client, fmt.Sprintf("%s/local-ipv4", ms.metadataUrl())); err == nil {
|
||||
attrs["local-ipv4"] = localAddr
|
||||
} else if _, ok := err.(pkg.ErrNotFound); !ok {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if publicAddr, err := fetchAttribute(ms.client, fmt.Sprintf("%s/public-ipv4", ms.metadataUrl())); err == nil {
|
||||
attrs["public-ipv4"] = publicAddr
|
||||
} else if _, ok := err.(pkg.ErrNotFound); !ok {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if content_path, err := fetchAttribute(ms.client, fmt.Sprintf("%s/network_config/content_path", ms.metadataUrl())); err == nil {
|
||||
attrs["network_config"] = map[string]string{
|
||||
"content_path": content_path,
|
||||
}
|
||||
} else if _, ok := err.(pkg.ErrNotFound); !ok {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return json.Marshal(attrs)
|
||||
}
|
||||
|
||||
func (ms metadataService) FetchUserdata() ([]byte, error) {
|
||||
if data, err := ms.client.GetRetry(ms.userdataUrl()); err == nil {
|
||||
return data, err
|
||||
} else if _, ok := err.(pkg.ErrNotFound); ok {
|
||||
return []byte{}, nil
|
||||
} else {
|
||||
return data, err
|
||||
}
|
||||
}
|
||||
|
||||
func (ms metadataService) Type() string {
|
||||
return "ec2-metadata-service"
|
||||
}
|
||||
|
||||
func (ms metadataService) metadataUrl() string {
|
||||
return (ms.root + metadataUrl)
|
||||
}
|
||||
|
||||
func (ms metadataService) userdataUrl() string {
|
||||
return (ms.root + userdataUrl)
|
||||
}
|
||||
|
||||
func fetchAttributes(client pkg.Getter, url string) ([]string, error) {
|
||||
resp, err := client.GetRetry(url)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
scanner := bufio.NewScanner(bytes.NewBuffer(resp))
|
||||
data := make([]string, 0)
|
||||
for scanner.Scan() {
|
||||
data = append(data, scanner.Text())
|
||||
}
|
||||
return data, scanner.Err()
|
||||
}
|
||||
|
||||
func fetchAttribute(client pkg.Getter, url string) (string, error) {
|
||||
if attrs, err := fetchAttributes(client, url); err == nil && len(attrs) > 0 {
|
||||
return attrs[0], nil
|
||||
} else {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
324
datasource/metadata/ec2/metadata_test.go
Normal file
324
datasource/metadata/ec2/metadata_test.go
Normal file
@@ -0,0 +1,324 @@
|
||||
package ec2
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"github.com/coreos/coreos-cloudinit/pkg"
|
||||
)
|
||||
|
||||
type testHttpClient struct {
|
||||
resources map[string]string
|
||||
err error
|
||||
}
|
||||
|
||||
func (t *testHttpClient) GetRetry(url string) ([]byte, error) {
|
||||
if t.err != nil {
|
||||
return nil, t.err
|
||||
}
|
||||
if val, ok := t.resources[url]; ok {
|
||||
return []byte(val), nil
|
||||
} else {
|
||||
return nil, pkg.ErrNotFound{fmt.Errorf("not found: %q", url)}
|
||||
}
|
||||
}
|
||||
|
||||
func (t *testHttpClient) Get(url string) ([]byte, error) {
|
||||
return t.GetRetry(url)
|
||||
}
|
||||
|
||||
func TestAvailabilityChanges(t *testing.T) {
|
||||
want := true
|
||||
if ac := (metadataService{}).AvailabilityChanges(); ac != want {
|
||||
t.Fatalf("bad AvailabilityChanges: want %q, got %q", want, ac)
|
||||
}
|
||||
}
|
||||
|
||||
func TestType(t *testing.T) {
|
||||
want := "ec2-metadata-service"
|
||||
if kind := (metadataService{}).Type(); kind != want {
|
||||
t.Fatalf("bad type: want %q, got %q", want, kind)
|
||||
}
|
||||
}
|
||||
|
||||
func TestIsAvailable(t *testing.T) {
|
||||
for _, tt := range []struct {
|
||||
root string
|
||||
resources map[string]string
|
||||
expect bool
|
||||
}{
|
||||
{
|
||||
root: "/",
|
||||
resources: map[string]string{
|
||||
"/2009-04-04": "",
|
||||
},
|
||||
expect: true,
|
||||
},
|
||||
{
|
||||
root: "/",
|
||||
resources: map[string]string{},
|
||||
expect: false,
|
||||
},
|
||||
} {
|
||||
service := &metadataService{tt.root, &testHttpClient{tt.resources, nil}}
|
||||
if a := service.IsAvailable(); a != tt.expect {
|
||||
t.Fatalf("bad isAvailable (%q): want %q, got %q", tt.resources, tt.expect, a)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetchUserdata(t *testing.T) {
|
||||
for _, tt := range []struct {
|
||||
root string
|
||||
resources map[string]string
|
||||
userdata []byte
|
||||
clientErr error
|
||||
expectErr error
|
||||
}{
|
||||
{
|
||||
root: "/",
|
||||
resources: map[string]string{
|
||||
"/2009-04-04/user-data": "hello",
|
||||
},
|
||||
userdata: []byte("hello"),
|
||||
},
|
||||
{
|
||||
root: "/",
|
||||
clientErr: pkg.ErrNotFound{fmt.Errorf("test not found error")},
|
||||
userdata: []byte{},
|
||||
},
|
||||
{
|
||||
root: "/",
|
||||
clientErr: pkg.ErrTimeout{fmt.Errorf("test timeout error")},
|
||||
expectErr: pkg.ErrTimeout{fmt.Errorf("test timeout error")},
|
||||
},
|
||||
} {
|
||||
service := &metadataService{tt.root, &testHttpClient{tt.resources, tt.clientErr}}
|
||||
data, err := service.FetchUserdata()
|
||||
if Error(err) != Error(tt.expectErr) {
|
||||
t.Fatalf("bad error (%q): want %q, got %q", tt.resources, tt.expectErr, err)
|
||||
}
|
||||
if !bytes.Equal(data, tt.userdata) {
|
||||
t.Fatalf("bad userdata (%q): want %q, got %q", tt.resources, tt.userdata, data)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestUrls(t *testing.T) {
|
||||
for _, tt := range []struct {
|
||||
root string
|
||||
expectRoot string
|
||||
userdata string
|
||||
metadata string
|
||||
}{
|
||||
{
|
||||
root: "/",
|
||||
expectRoot: "/",
|
||||
userdata: "/2009-04-04/user-data",
|
||||
metadata: "/2009-04-04/meta-data",
|
||||
},
|
||||
{
|
||||
root: "http://169.254.169.254/",
|
||||
expectRoot: "http://169.254.169.254/",
|
||||
userdata: "http://169.254.169.254/2009-04-04/user-data",
|
||||
metadata: "http://169.254.169.254/2009-04-04/meta-data",
|
||||
},
|
||||
} {
|
||||
service := &metadataService{tt.root, nil}
|
||||
if url := service.userdataUrl(); url != tt.userdata {
|
||||
t.Fatalf("bad url (%q): want %q, got %q", tt.root, tt.userdata, url)
|
||||
}
|
||||
if url := service.metadataUrl(); url != tt.metadata {
|
||||
t.Fatalf("bad url (%q): want %q, got %q", tt.root, tt.metadata, url)
|
||||
}
|
||||
if url := service.ConfigRoot(); url != tt.expectRoot {
|
||||
t.Fatalf("bad url (%q): want %q, got %q", tt.root, tt.expectRoot, url)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetchAttributes(t *testing.T) {
|
||||
for _, s := range []struct {
|
||||
resources map[string]string
|
||||
err error
|
||||
tests []struct {
|
||||
path string
|
||||
val []string
|
||||
}
|
||||
}{
|
||||
{
|
||||
resources: map[string]string{
|
||||
"/": "a\nb\nc/",
|
||||
"/c/": "d\ne/",
|
||||
"/c/e/": "f",
|
||||
"/a": "1",
|
||||
"/b": "2",
|
||||
"/c/d": "3",
|
||||
"/c/e/f": "4",
|
||||
},
|
||||
tests: []struct {
|
||||
path string
|
||||
val []string
|
||||
}{
|
||||
{"/", []string{"a", "b", "c/"}},
|
||||
{"/b", []string{"2"}},
|
||||
{"/c/d", []string{"3"}},
|
||||
{"/c/e/", []string{"f"}},
|
||||
},
|
||||
},
|
||||
{
|
||||
err: pkg.ErrNotFound{fmt.Errorf("test error")},
|
||||
tests: []struct {
|
||||
path string
|
||||
val []string
|
||||
}{
|
||||
{"", nil},
|
||||
},
|
||||
},
|
||||
} {
|
||||
client := &testHttpClient{s.resources, s.err}
|
||||
for _, tt := range s.tests {
|
||||
attrs, err := fetchAttributes(client, tt.path)
|
||||
if err != s.err {
|
||||
t.Fatalf("bad error for %q (%q): want %q, got %q", tt.path, s.resources, s.err, err)
|
||||
}
|
||||
if !reflect.DeepEqual(attrs, tt.val) {
|
||||
t.Fatalf("bad fetch for %q (%q): want %q, got %q", tt.path, s.resources, tt.val, attrs)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetchAttribute(t *testing.T) {
|
||||
for _, s := range []struct {
|
||||
resources map[string]string
|
||||
err error
|
||||
tests []struct {
|
||||
path string
|
||||
val string
|
||||
}
|
||||
}{
|
||||
{
|
||||
resources: map[string]string{
|
||||
"/": "a\nb\nc/",
|
||||
"/c/": "d\ne/",
|
||||
"/c/e/": "f",
|
||||
"/a": "1",
|
||||
"/b": "2",
|
||||
"/c/d": "3",
|
||||
"/c/e/f": "4",
|
||||
},
|
||||
tests: []struct {
|
||||
path string
|
||||
val string
|
||||
}{
|
||||
{"/a", "1"},
|
||||
{"/b", "2"},
|
||||
{"/c/d", "3"},
|
||||
{"/c/e/f", "4"},
|
||||
},
|
||||
},
|
||||
{
|
||||
err: pkg.ErrNotFound{fmt.Errorf("test error")},
|
||||
tests: []struct {
|
||||
path string
|
||||
val string
|
||||
}{
|
||||
{"", ""},
|
||||
},
|
||||
},
|
||||
} {
|
||||
client := &testHttpClient{s.resources, s.err}
|
||||
for _, tt := range s.tests {
|
||||
attr, err := fetchAttribute(client, tt.path)
|
||||
if err != s.err {
|
||||
t.Fatalf("bad error for %q (%q): want %q, got %q", tt.path, s.resources, s.err, err)
|
||||
}
|
||||
if attr != tt.val {
|
||||
t.Fatalf("bad fetch for %q (%q): want %q, got %q", tt.path, s.resources, tt.val, attr)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestFetchMetadata(t *testing.T) {
|
||||
for _, tt := range []struct {
|
||||
root string
|
||||
resources map[string]string
|
||||
expect []byte
|
||||
clientErr error
|
||||
expectErr error
|
||||
}{
|
||||
{
|
||||
root: "/",
|
||||
resources: map[string]string{
|
||||
"/2009-04-04/meta-data/public-keys": "bad\n",
|
||||
},
|
||||
expectErr: fmt.Errorf("malformed public key: \"bad\""),
|
||||
},
|
||||
{
|
||||
root: "/",
|
||||
resources: map[string]string{
|
||||
"/2009-04-04/meta-data/hostname": "host",
|
||||
"/2009-04-04/meta-data/local-ipv4": "1.2.3.4",
|
||||
"/2009-04-04/meta-data/public-ipv4": "5.6.7.8",
|
||||
"/2009-04-04/meta-data/public-keys": "0=test1\n",
|
||||
"/2009-04-04/meta-data/public-keys/0": "openssh-key",
|
||||
"/2009-04-04/meta-data/public-keys/0/openssh-key": "key",
|
||||
"/2009-04-04/meta-data/network_config/content_path": "path",
|
||||
},
|
||||
expect: []byte(`{"hostname":"host","local-ipv4":"1.2.3.4","network_config":{"content_path":"path"},"public-ipv4":"5.6.7.8","public_keys":{"test1":"key"}}`),
|
||||
},
|
||||
{
|
||||
clientErr: pkg.ErrTimeout{fmt.Errorf("test error")},
|
||||
expectErr: pkg.ErrTimeout{fmt.Errorf("test error")},
|
||||
},
|
||||
} {
|
||||
service := &metadataService{tt.root, &testHttpClient{tt.resources, tt.clientErr}}
|
||||
metadata, err := service.FetchMetadata()
|
||||
if Error(err) != Error(tt.expectErr) {
|
||||
t.Fatalf("bad error (%q): want %q, got %q", tt.resources, tt.expectErr, err)
|
||||
}
|
||||
if !bytes.Equal(metadata, tt.expect) {
|
||||
t.Fatalf("bad fetch (%q): want %q, got %q", tt.resources, tt.expect, metadata)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewDatasource(t *testing.T) {
|
||||
for _, tt := range []struct {
|
||||
root string
|
||||
expectRoot string
|
||||
}{
|
||||
{
|
||||
root: "",
|
||||
expectRoot: "/",
|
||||
},
|
||||
{
|
||||
root: "/",
|
||||
expectRoot: "/",
|
||||
},
|
||||
{
|
||||
root: "http://169.254.169.254",
|
||||
expectRoot: "http://169.254.169.254/",
|
||||
},
|
||||
{
|
||||
root: "http://169.254.169.254/",
|
||||
expectRoot: "http://169.254.169.254/",
|
||||
},
|
||||
} {
|
||||
service := NewDatasource(tt.root)
|
||||
if service.root != tt.expectRoot {
|
||||
t.Fatalf("bad root (%q): want %q, got %q", tt.root, tt.expectRoot, service.root)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func Error(err error) string {
|
||||
if err != nil {
|
||||
return err.Error()
|
||||
}
|
||||
return ""
|
||||
}
|
||||
@@ -1,153 +0,0 @@
|
||||
package datasource
|
||||
|
||||
import (
|
||||
"bufio"
|
||||
"bytes"
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"strings"
|
||||
|
||||
"github.com/coreos/coreos-cloudinit/pkg"
|
||||
)
|
||||
|
||||
// metadataService retrieves metadata from either an OpenStack[1] (2012-08-10)
|
||||
// or EC2[2] (2009-04-04) compatible endpoint. It will first attempt to
|
||||
// directly retrieve a JSON blob from the OpenStack endpoint. If that fails
|
||||
// with a 404, it then attempts to retrieve metadata bit-by-bit from the EC2
|
||||
// endpoint, and populates that into an equivalent JSON blob. metadataService
|
||||
// also checks for userdata from EC2 and, if that fails with a 404, OpenStack.
|
||||
//
|
||||
// [1] http://docs.openstack.org/grizzly/openstack-compute/admin/content/metadata-service.html
|
||||
// [2] http://docs.aws.amazon.com/AWSEC2/latest/UserGuide/AESDG-chapter-instancedata.html#instancedata-data-categories
|
||||
|
||||
const (
|
||||
BaseUrl = "http://169.254.169.254/"
|
||||
Ec2UserdataUrl = BaseUrl + Ec2ApiVersion + "/user-data"
|
||||
Ec2MetadataUrl = BaseUrl + Ec2ApiVersion + "/meta-data"
|
||||
OpenstackUserdataUrl = BaseUrl + "openstack/" + OpenstackApiVersion + "/user_data"
|
||||
)
|
||||
|
||||
type metadataService struct{}
|
||||
|
||||
type getter interface {
|
||||
GetRetry(string) ([]byte, error)
|
||||
}
|
||||
|
||||
func NewMetadataService() *metadataService {
|
||||
return &metadataService{}
|
||||
}
|
||||
|
||||
func (ms *metadataService) IsAvailable() bool {
|
||||
client := pkg.NewHttpClient()
|
||||
_, err := client.Get(BaseUrl)
|
||||
return (err == nil)
|
||||
}
|
||||
|
||||
func (ms *metadataService) AvailabilityChanges() bool {
|
||||
return true
|
||||
}
|
||||
|
||||
func (ms *metadataService) ConfigRoot() string {
|
||||
return ""
|
||||
}
|
||||
|
||||
func (ms *metadataService) FetchMetadata() ([]byte, error) {
|
||||
return fetchMetadata(pkg.NewHttpClient())
|
||||
}
|
||||
|
||||
func (ms *metadataService) FetchUserdata() ([]byte, error) {
|
||||
client := pkg.NewHttpClient()
|
||||
if data, err := client.GetRetry(Ec2UserdataUrl); err == nil {
|
||||
return data, err
|
||||
} else if _, ok := err.(pkg.ErrTimeout); ok {
|
||||
return data, err
|
||||
}
|
||||
|
||||
if data, err := client.GetRetry(OpenstackUserdataUrl); err == nil {
|
||||
return data, err
|
||||
} else if _, ok := err.(pkg.ErrNotFound); ok {
|
||||
return []byte{}, nil
|
||||
} else {
|
||||
return data, err
|
||||
}
|
||||
}
|
||||
|
||||
func (ms *metadataService) Type() string {
|
||||
return "metadata-service"
|
||||
}
|
||||
|
||||
func fetchMetadata(client getter) ([]byte, error) {
|
||||
attrs := make(map[string]interface{})
|
||||
if keynames, err := fetchAttributes(client, fmt.Sprintf("%s/public-keys", Ec2MetadataUrl)); err == nil {
|
||||
keyIDs := make(map[string]string)
|
||||
for _, keyname := range keynames {
|
||||
tokens := strings.SplitN(keyname, "=", 2)
|
||||
if len(tokens) != 2 {
|
||||
return nil, fmt.Errorf("malformed public key: %q\n", keyname)
|
||||
}
|
||||
keyIDs[tokens[1]] = tokens[0]
|
||||
}
|
||||
|
||||
keys := make(map[string]string)
|
||||
for name, id := range keyIDs {
|
||||
sshkey, err := fetchAttribute(client, fmt.Sprintf("%s/public-keys/%s/openssh-key", Ec2MetadataUrl, id))
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
keys[name] = sshkey
|
||||
fmt.Printf("Found SSH key for %q\n", name)
|
||||
}
|
||||
attrs["public_keys"] = keys
|
||||
} else if _, ok := err.(pkg.ErrNotFound); !ok {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if hostname, err := fetchAttribute(client, fmt.Sprintf("%s/hostname", Ec2MetadataUrl)); err == nil {
|
||||
attrs["hostname"] = hostname
|
||||
} else if _, ok := err.(pkg.ErrNotFound); !ok {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if localAddr, err := fetchAttribute(client, fmt.Sprintf("%s/local-ipv4", Ec2MetadataUrl)); err == nil {
|
||||
attrs["local-ipv4"] = localAddr
|
||||
} else if _, ok := err.(pkg.ErrNotFound); !ok {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if publicAddr, err := fetchAttribute(client, fmt.Sprintf("%s/public-ipv4", Ec2MetadataUrl)); err == nil {
|
||||
attrs["public-ipv4"] = publicAddr
|
||||
} else if _, ok := err.(pkg.ErrNotFound); !ok {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
if content_path, err := fetchAttribute(client, fmt.Sprintf("%s/network_config/content_path", Ec2MetadataUrl)); err == nil {
|
||||
attrs["network_config"] = map[string]string{
|
||||
"content_path": content_path,
|
||||
}
|
||||
} else if _, ok := err.(pkg.ErrNotFound); !ok {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
return json.Marshal(attrs)
|
||||
}
|
||||
|
||||
func fetchAttributes(client getter, url string) ([]string, error) {
|
||||
resp, err := client.GetRetry(url)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
scanner := bufio.NewScanner(bytes.NewBuffer(resp))
|
||||
data := make([]string, 0)
|
||||
for scanner.Scan() {
|
||||
data = append(data, scanner.Text())
|
||||
}
|
||||
return data, scanner.Err()
|
||||
}
|
||||
|
||||
func fetchAttribute(client getter, url string) (string, error) {
|
||||
if attrs, err := fetchAttributes(client, url); err == nil && len(attrs) > 0 {
|
||||
return attrs[0], nil
|
||||
} else {
|
||||
return "", err
|
||||
}
|
||||
}
|
||||
@@ -1,159 +0,0 @@
|
||||
package datasource
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"fmt"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"github.com/coreos/coreos-cloudinit/pkg"
|
||||
)
|
||||
|
||||
type TestHttpClient struct {
|
||||
metadata map[string]string
|
||||
err error
|
||||
}
|
||||
|
||||
func (t *TestHttpClient) GetRetry(url string) ([]byte, error) {
|
||||
if t.err != nil {
|
||||
return nil, t.err
|
||||
}
|
||||
if val, ok := t.metadata[url]; ok {
|
||||
return []byte(val), nil
|
||||
} else {
|
||||
return nil, pkg.ErrNotFound{fmt.Errorf("not found: %q", url)}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMSFetchAttributes(t *testing.T) {
|
||||
for _, s := range []struct {
|
||||
metadata map[string]string
|
||||
err error
|
||||
tests []struct {
|
||||
path string
|
||||
val []string
|
||||
}
|
||||
}{
|
||||
{
|
||||
metadata: map[string]string{
|
||||
"/": "a\nb\nc/",
|
||||
"/c/": "d\ne/",
|
||||
"/c/e/": "f",
|
||||
"/a": "1",
|
||||
"/b": "2",
|
||||
"/c/d": "3",
|
||||
"/c/e/f": "4",
|
||||
},
|
||||
tests: []struct {
|
||||
path string
|
||||
val []string
|
||||
}{
|
||||
{"/", []string{"a", "b", "c/"}},
|
||||
{"/b", []string{"2"}},
|
||||
{"/c/d", []string{"3"}},
|
||||
{"/c/e/", []string{"f"}},
|
||||
},
|
||||
},
|
||||
{
|
||||
err: pkg.ErrNotFound{fmt.Errorf("test error")},
|
||||
tests: []struct {
|
||||
path string
|
||||
val []string
|
||||
}{
|
||||
{"", nil},
|
||||
},
|
||||
},
|
||||
} {
|
||||
client := &TestHttpClient{s.metadata, s.err}
|
||||
for _, tt := range s.tests {
|
||||
attrs, err := fetchAttributes(client, tt.path)
|
||||
if err != s.err {
|
||||
t.Fatalf("bad error for %q (%q): want %q, got %q", tt.path, s.metadata, s.err, err)
|
||||
}
|
||||
if !reflect.DeepEqual(attrs, tt.val) {
|
||||
t.Fatalf("bad fetch for %q (%q): want %q, got %q", tt.path, s.metadata, tt.val, attrs)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMSFetchAttribute(t *testing.T) {
|
||||
for _, s := range []struct {
|
||||
metadata map[string]string
|
||||
err error
|
||||
tests []struct {
|
||||
path string
|
||||
val string
|
||||
}
|
||||
}{
|
||||
{
|
||||
metadata: map[string]string{
|
||||
"/": "a\nb\nc/",
|
||||
"/c/": "d\ne/",
|
||||
"/c/e/": "f",
|
||||
"/a": "1",
|
||||
"/b": "2",
|
||||
"/c/d": "3",
|
||||
"/c/e/f": "4",
|
||||
},
|
||||
tests: []struct {
|
||||
path string
|
||||
val string
|
||||
}{
|
||||
{"/a", "1"},
|
||||
{"/b", "2"},
|
||||
{"/c/d", "3"},
|
||||
{"/c/e/f", "4"},
|
||||
},
|
||||
},
|
||||
{
|
||||
err: pkg.ErrNotFound{fmt.Errorf("test error")},
|
||||
tests: []struct {
|
||||
path string
|
||||
val string
|
||||
}{
|
||||
{"", ""},
|
||||
},
|
||||
},
|
||||
} {
|
||||
client := &TestHttpClient{s.metadata, s.err}
|
||||
for _, tt := range s.tests {
|
||||
attr, err := fetchAttribute(client, tt.path)
|
||||
if err != s.err {
|
||||
t.Fatalf("bad error for %q (%q): want %q, got %q", tt.path, s.metadata, s.err, err)
|
||||
}
|
||||
if attr != tt.val {
|
||||
t.Fatalf("bad fetch for %q (%q): want %q, got %q", tt.path, s.metadata, tt.val, attr)
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func TestMSFetchMetadata(t *testing.T) {
|
||||
for _, tt := range []struct {
|
||||
metadata map[string]string
|
||||
err error
|
||||
expect []byte
|
||||
}{
|
||||
{
|
||||
metadata: map[string]string{
|
||||
"http://169.254.169.254/2009-04-04/meta-data/hostname": "host",
|
||||
"http://169.254.169.254/2009-04-04/meta-data/public-keys": "0=test1\n",
|
||||
"http://169.254.169.254/2009-04-04/meta-data/public-keys/0": "openssh-key",
|
||||
"http://169.254.169.254/2009-04-04/meta-data/public-keys/0/openssh-key": "key",
|
||||
"http://169.254.169.254/2009-04-04/meta-data/network_config/content_path": "path",
|
||||
},
|
||||
expect: []byte(`{"hostname":"host","network_config":{"content_path":"path"},"public_keys":{"test1":"key"}}`),
|
||||
},
|
||||
{err: pkg.ErrTimeout{fmt.Errorf("test error")}},
|
||||
} {
|
||||
client := &TestHttpClient{tt.metadata, tt.err}
|
||||
metadata, err := fetchMetadata(client)
|
||||
if err != tt.err {
|
||||
t.Fatalf("bad error (%q): want %q, got %q", tt.metadata, tt.err, err)
|
||||
}
|
||||
if !bytes.Equal(metadata, tt.expect) {
|
||||
t.Fatalf("bad fetch (%q): want %q, got %q", tt.metadata, tt.expect, metadata)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,4 +1,4 @@
|
||||
package datasource
|
||||
package proc_cmdline
|
||||
|
||||
import (
|
||||
"errors"
|
||||
@@ -18,7 +18,7 @@ type procCmdline struct {
|
||||
Location string
|
||||
}
|
||||
|
||||
func NewProcCmdline() *procCmdline {
|
||||
func NewDatasource() *procCmdline {
|
||||
return &procCmdline{Location: ProcCmdlineLocation}
|
||||
}
|
||||
|
||||
@@ -1,4 +1,4 @@
|
||||
package datasource
|
||||
package proc_cmdline
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
@@ -75,7 +75,7 @@ func TestProcCmdlineAndFetchConfig(t *testing.T) {
|
||||
t.Errorf("Test produced error: %v", err)
|
||||
}
|
||||
|
||||
p := NewProcCmdline()
|
||||
p := NewDatasource()
|
||||
p.Location = file.Name()
|
||||
cfg, err := p.FetchUserdata()
|
||||
if err != nil {
|
||||
@@ -1,4 +1,4 @@
|
||||
package datasource
|
||||
package url
|
||||
|
||||
import "github.com/coreos/coreos-cloudinit/pkg"
|
||||
|
||||
@@ -6,7 +6,7 @@ type remoteFile struct {
|
||||
url string
|
||||
}
|
||||
|
||||
func NewRemoteFile(url string) *remoteFile {
|
||||
func NewDatasource(url string) *remoteFile {
|
||||
return &remoteFile{url}
|
||||
}
|
||||
|
||||
@@ -57,6 +57,11 @@ type HttpClient struct {
|
||||
client *http.Client
|
||||
}
|
||||
|
||||
type Getter interface {
|
||||
Get(string) ([]byte, error)
|
||||
GetRetry(string) ([]byte, error)
|
||||
}
|
||||
|
||||
func NewHttpClient() *HttpClient {
|
||||
hc := &HttpClient{
|
||||
MaxBackoff: time.Second * 5,
|
||||
|
||||
11
test
11
test
@@ -13,7 +13,16 @@ COVER=${COVER:-"-cover"}
|
||||
|
||||
source ./build
|
||||
|
||||
declare -a TESTPKGS=(initialize system datasource pkg network)
|
||||
declare -a TESTPKGS=(initialize
|
||||
system
|
||||
datasource
|
||||
datasource/configdrive
|
||||
datasource/file
|
||||
datasource/metadata/ec2
|
||||
datasource/proc_cmdline
|
||||
datasource/url
|
||||
pkg
|
||||
network)
|
||||
|
||||
if [ -z "$PKG" ]; then
|
||||
GOFMTPATH="$TESTPKGS coreos-cloudinit.go"
|
||||
|
||||
Reference in New Issue
Block a user