system: some user inmprovements

* not use cgo
* add lock/unlock user

Signed-off-by: Vasiliy Tolstov <v.tolstov@selfip.ru>

Documentation/cloud-config-locations.md

@@ -9,13 +9,18 @@ Location | Description
 |Kernel command line: `cloud-config-url=http://example.com/user_data`.| You can find this string using this command `cat /proc/cmdline`. Usually used in [PXE](/os/docs/latest/booting-with-pxe.html) or [iPXE](/os/docs/latest/booting-with-ipxe.html) boots.|
 |`/var/lib/coreos-install/user_data`| When you install CoreOS manually using the [coreos-install](/os/docs/latest/installing-to-disk.html) tool. Usually used in bare metal installations.|
 |`/usr/share/oem/cloud-config.yml`| Path for OEM images.|
+|`/var/lib/coreos-vagrant/vagrantfile-user-data`| Vagrant OEM scripts automatically store Cloud-Config into this path. |
 |`/var/lib/waagent/CustomData`| Azure platform uses OEM path for first Cloud-Config initialization and then `/var/lib/waagent/CustomData` to apply your settings.|
 |`http://169.254.169.254/metadata/v1/user-data` `http://169.254.169.254/2009-04-04/user-data` `https://metadata.packet.net/userdata`|DigitalOcean, EC2 and Packet cloud providers correspondingly use these URLs to download Cloud-Config.|
+|`/usr/share/oem/bin/vmtoolsd --cmd "info-get guestinfo.coreos.config.data"`|Cloud-Config provided by [VMware Guestinfo][VMware Guestinfo]|
+|`/usr/share/oem/bin/vmtoolsd --cmd "info-get guestinfo.coreos.config.url"`|Cloud-Config URL provided by [VMware Guestinfo][VMware Guestinfo]|
+
+[VMware Guestinfo]: vmware-guestinfo.md
 
 You can also run the `coreos-cloudinit` tool manually and provide a path to your custom Cloud-Config file:
 
 ```sh
 sudo coreos-cloudinit --from-file=/home/core/cloud-config.yaml
-``` 
+```
 
 This command will apply your custom cloud-config.

Documentation/cloud-config.md

@@ -81,7 +81,7 @@ For more information about the available configuration parameters, see the [etcd
 
 _Note: The `$private_ipv4` and `$public_ipv4` substitution variables referenced in other documents are only supported on Amazon EC2, Google Compute Engine, OpenStack, Rackspace, DigitalOcean, and Vagrant._
 
-[etcd-config]: https://github.com/coreos/etcd/blob/9fa3bea5a22265151f0d5063ce38a79c5b5d0271/Documentation/configuration.md
+[etcd-config]: https://github.com/coreos/etcd/blob/release-0.4/Documentation/configuration.md
 
 #### etcd2
 
@@ -117,11 +117,11 @@ Environment="ETCD_LISTEN_CLIENT_URLS=http://0.0.0.0:2379,http://0.0.0.0:4001"
 Environment="ETCD_LISTEN_PEER_URLS=http://192.0.2.13:2380,http://192.0.2.13:7001"
 ```
 
-For more information about the available configuration parameters, see the [etcd documentation][etcd-config].
+For more information about the available configuration parameters, see the [etcd2 documentation][etcd2-config].
 
 _Note: The `$private_ipv4` and `$public_ipv4` substitution variables referenced in other documents are only supported on Amazon EC2, Google Compute Engine, OpenStack, Rackspace, DigitalOcean, and Vagrant._
 
-[etcd-config]: https://github.com/coreos/etcd/blob/86e616c6e974828fc9119c1eb0f6439577a9ce0b/Documentation/configuration.md
+[etcd2-config]: https://github.com/coreos/etcd/blob/master/Documentation/configuration.md
 
 #### fleet
 

Documentation/vmware-guestinfo.md

@@ -1,12 +1,16 @@
-# VMware Backdoor #
+# VMWare Guestinfo Interface
 
-coreos-cloudinit is capable of reading userdata and metadata from the VMware
-backdoor. This datasource can be enable with the `--from-vmware-backdoor` flag.
-Userdata and metadata are passed from the hypervisor to the virtual machine
-through guest variables. The following guest variables and their expected types
-are supported by coreos-cloudinit:
+## Cloud-Config VMWare Guestinfo Variables
 
-|            guest variable             |              type               |
+coreos-cloudinit accepts configuration from the VMware RPC API's *guestinfo*
+facility. This datasource can be enabled with the `--from-vmware-guestinfo`
+flag to coreos-cloudinit.
+
+The following guestinfo variables are recognized and processed by cloudinit
+when passed from the hypervisor to the virtual machine at boot time. Note that
+property names are prefixed with `guestinfo.` in the VMX, e.g., `guestinfo.hostname`.
+
+|            guestinfo variable             |              type               |
 |:--------------------------------------|:--------------------------------|
 | `hostname`                            | `hostname`                      |
 | `interface.<n>.name`                  | `string`                        |
@@ -22,5 +26,10 @@ are supported by coreos-cloudinit:
 | `coreos.config.url`                   | `URL`                           |
 
 Note: "n", "m", "l", and "x" are 0-indexed, incrementing integers. The
-identifier for the interfaces does not correspond to anything outside of this
-configuration; it is merely for mapping configuration values to each interface.
+identifier for an `interface` does not correspond to anything outside of this
+configuration; it serves only to distinguish between multiple `interface`s.
+
+The guide to [booting on VMWare][bootvmware] is the starting point for more
+information about configuring and running CoreOS on VMWare.
+
+[bootvmware]: https://github.com/coreos/docs/blob/master/os/booting-on-vmware.md

config/config_test.go

@@ -164,7 +164,7 @@ func TestConfigCompile(t *testing.T) {
 
 func TestCloudConfigUnknownKeys(t *testing.T) {
 	contents := `
-coreos: 
+coreos:
   etcd:
     discovery: "https://discovery.etcd.io/827c73219eeb2fa5530027c37bf18877"
   coreos_unknown:
@@ -227,7 +227,7 @@ func TestCloudConfigEmpty(t *testing.T) {
 // Assert that the parsing of a cloud config file "generally works"
 func TestCloudConfig(t *testing.T) {
 	contents := `
-coreos: 
+coreos:
   etcd:
     discovery: "https://discovery.etcd.io/827c73219eeb2fa5530027c37bf18877"
   update:
@@ -236,14 +236,14 @@ coreos:
     - name: 50-eth0.network
       runtime: yes
       content: '[Match]
- 
+
     Name=eth47
- 
- 
+
+
     [Network]
- 
+
     Address=10.209.171.177/19
- 
+
 '
   oem:
     id: rackspace
@@ -367,6 +367,7 @@ users:
     gecos: arbitrary comment
     homedir: /home/place
     no_create_home: yes
+    lock_passwd: false
     primary_group: things
     groups:
       - ping

config/etcd2.go

@@ -53,4 +53,5 @@ type Etcd2 struct {
 	ProxyWriteTimeout        int    `yaml:"proxy_write_timeout"           env:"ETCD_PROXY_WRITE_TIMEOUT"`
 	SnapshotCount            int    `yaml:"snapshot_count"                env:"ETCD_SNAPSHOT_COUNT"`
 	TrustedCAFile            string `yaml:"trusted_ca_file"               env:"ETCD_TRUSTED_CA_FILE"`
+	WalDir                   string `yaml:"wal_dir"                       env:"ETCD_WAL_DIR"`
 }

config/fleet.go

@@ -16,6 +16,8 @@ package config
 
 type Fleet struct {
 	AgentTTL                string  `yaml:"agent_ttl"                 env:"FLEET_AGENT_TTL"`
+	AuthorizedKeysFile      string  `yaml:"authorized_keys_file"      env:"FLEET_AUTHORIZED_KEYS_FILE"`
+	DisableEngine           bool    `yaml:"disable_engine"            env:"FLEET_DISABLE_ENGINE"`
 	EngineReconcileInterval float64 `yaml:"engine_reconcile_interval" env:"FLEET_ENGINE_RECONCILE_INTERVAL"`
 	EtcdCAFile              string  `yaml:"etcd_cafile"               env:"FLEET_ETCD_CAFILE"`
 	EtcdCertFile            string  `yaml:"etcd_certfile"             env:"FLEET_ETCD_CERTFILE"`
@@ -25,5 +27,7 @@ type Fleet struct {
 	EtcdServers             string  `yaml:"etcd_servers"              env:"FLEET_ETCD_SERVERS"`
 	Metadata                string  `yaml:"metadata"                  env:"FLEET_METADATA"`
 	PublicIP                string  `yaml:"public_ip"                 env:"FLEET_PUBLIC_IP"`
+	TokenLimit              int     `yaml:"token_limit"               env:"FLEET_TOKEN_LIMIT"`
 	Verbosity               int     `yaml:"verbosity"                 env:"FLEET_VERBOSITY"`
+	VerifyUnits             bool    `yaml:"verify_units"              env:"FLEET_VERIFY_UNITS"`
 }

config/user.go

@@ -24,6 +24,7 @@ type User struct {
 	GECOS                string   `yaml:"gecos"`
 	Homedir              string   `yaml:"homedir"`
 	NoCreateHome         bool     `yaml:"no_create_home"`
+	LockPasswd           bool     `yaml:"lock_passwd"`
 	PrimaryGroup         string   `yaml:"primary_group"`
 	Groups               []string `yaml:"groups"`
 	NoUserGroup          bool     `yaml:"no_user_group"`

coreos-cloudinit.go

@@ -15,8 +15,11 @@
 package main
 
 import (
+	"bytes"
+	"compress/gzip"
 	"flag"
 	"fmt"
+	"io/ioutil"
 	"log"
 	"os"
 	"runtime"
@@ -87,7 +90,7 @@ func init() {
 	flag.StringVar(&flags.sources.packetMetadataService, "from-packet-metadata", "", "Download Packet data from metadata service")
 	flag.StringVar(&flags.sources.url, "from-url", "", "Download user-data from provided url")
 	flag.BoolVar(&flags.sources.procCmdLine, "from-proc-cmdline", false, fmt.Sprintf("Parse %s for '%s=<url>', using the cloud-config served by an HTTP GET to <url>", proc_cmdline.ProcCmdlineLocation, proc_cmdline.ProcCmdlineCloudConfigFlag))
-	flag.BoolVar(&flags.sources.vmware, "from-vmware-backdoor", false, "Read data from VMware backdoor")
+	flag.BoolVar(&flags.sources.vmware, "from-vmware-guestinfo", false, "Read data from VMware guestinfo")
 	flag.StringVar(&flags.oem, "oem", "", "Use the settings specific to the provided OEM")
 	flag.StringVar(&flags.convertNetconf, "convert-netconf", "", "Read the network config provided in cloud-drive and translate it from the specified format into networkd unit files")
 	flag.StringVar(&flags.workspace, "workspace", "/var/lib/coreos-cloudinit", "Base directory coreos-cloudinit should use to store data")
@@ -121,7 +124,7 @@ var (
 			"from-packet-metadata": "https://metadata.packet.net/",
 		},
 		"vmware": oemConfig{
-			"from-vmware-backdoor": "true",
+			"from-vmware-guestinfo": "true",
 			"convert-netconf":      "vmware",
 		},
 	}
@@ -147,7 +150,7 @@ func main() {
 		for k := range oemConfigs {
 			oems = append(oems, k)
 		}
-		fmt.Printf("Invalid option to --oem: %q. Supported options: %q\n", flags.oem, oems)
+		fmt.Printf("Invalid option to -oem: %q. Supported options: %q\n", flags.oem, oems)
 		os.Exit(2)
 	}
 
@@ -169,7 +172,7 @@ func main() {
 
 	dss := getDatasources()
 	if len(dss) == 0 {
-		fmt.Println("Provide at least one of --from-file, --from-configdrive, --from-ec2-metadata, --from-cloudsigma-metadata, --from-packet-metadata, --from-url or --from-proc-cmdline")
+		fmt.Println("Provide at least one of --from-file, --from-configdrive, --from-ec2-metadata, --from-cloudsigma-metadata, --from-packet-metadata, --from-digitalocean-metadata, --from-vmware-guestinfo, --from-waagent, --from-url or --from-proc-cmdline")
 		os.Exit(2)
 	}
 
@@ -185,6 +188,11 @@ func main() {
 		log.Printf("Failed fetching user-data from datasource: %v. Continuing...\n", err)
 		failure = true
 	}
+	userdataBytes, err = decompressIfGzip(userdataBytes)
+	if err != nil {
+		log.Printf("Failed decompressing user-data from datasource: %v. Continuing...\n", err)
+		failure = true
+	}
 
 	if report, err := validate.Validate(userdataBytes); err == nil {
 		ret := 0
@@ -399,3 +407,17 @@ func runScript(script config.Script, env *initialize.Environment) error {
 	}
 	return err
 }
+
+const gzipMagicBytes = "\x1f\x8b"
+
+func decompressIfGzip(userdataBytes []byte) ([]byte, error) {
+	if !bytes.HasPrefix(userdataBytes, []byte(gzipMagicBytes)) {
+		return userdataBytes, nil
+	}
+	gzr, err := gzip.NewReader(bytes.NewReader(userdataBytes))
+	if err != nil {
+		return nil, err
+	}
+	defer gzr.Close()
+	return ioutil.ReadAll(gzr)
+}

coreos-cloudinit_test.go

@@ -15,6 +15,9 @@
 package main
 
 import (
+	"bytes"
+	"encoding/base64"
+	"errors"
 	"reflect"
 	"testing"
 
@@ -87,3 +90,58 @@ func TestMergeConfigs(t *testing.T) {
 		}
 	}
 }
+
+func mustDecode(in string) []byte {
+	out, err := base64.StdEncoding.DecodeString(in)
+	if err != nil {
+		panic(err)
+	}
+	return out
+}
+
+func TestDecompressIfGzip(t *testing.T) {
+	tests := []struct {
+		in []byte
+
+		out []byte
+		err error
+	}{
+		{
+			in: nil,
+
+			out: nil,
+			err: nil,
+		},
+		{
+			in: []byte{},
+
+			out: []byte{},
+			err: nil,
+		},
+		{
+			in: mustDecode("H4sIAJWV/VUAA1NOzskvTdFNzs9Ly0wHABt6mQENAAAA"),
+
+			out: []byte("#cloud-config"),
+			err: nil,
+		},
+		{
+			in: []byte("#cloud-config"),
+
+			out: []byte("#cloud-config"),
+			err: nil,
+		},
+		{
+			in: mustDecode("H4sCORRUPT=="),
+
+			out: nil,
+			err: errors.New("any error"),
+		},
+	}
+	for i, tt := range tests {
+		out, err := decompressIfGzip(tt.in)
+		if !bytes.Equal(out, tt.out) || (tt.err != nil && err == nil) {
+			t.Errorf("bad gzip (%d): want (%s, %#v), got (%s, %#v)", i, string(tt.out), tt.err, string(out), err)
+		}
+	}
+
+}

datasource/metadata/digitalocean/metadata.go

@@ -38,10 +38,11 @@ type Address struct {
 }
 
 type Interface struct {
-	IPv4 *Address `json:"ipv4"`
-	IPv6 *Address `json:"ipv6"`
-	MAC  string   `json:"mac"`
-	Type string   `json:"type"`
+	IPv4       *Address `json:"ipv4"`
+	IPv6       *Address `json:"ipv6"`
+	AnchorIPv4 *Address `json:"anchor_ipv4"`
+	MAC        string   `json:"mac"`
+	Type       string   `json:"type"`
 }
 
 type Interfaces struct {

initialize/config.go

@@ -73,6 +73,11 @@ func Apply(cfg config.CloudConfig, ifaces []network.InterfaceGenerator, env *Env
 			}
 		}
 
+		if err = system.LockUnlockUser(&user); err != nil {
+			log.Printf("Failed lock/unlock user '%s': %v", user.Name, err)
+			return err
+		}
+
 		if len(user.SSHAuthorizedKeys) > 0 {
 			log.Printf("Authorizing %d SSH keys for user '%s'", len(user.SSHAuthorizedKeys), user.Name)
 			if err := system.AuthorizeSSHKeys(user.Name, env.SSHKeyName(), user.SSHAuthorizedKeys); err != nil {

network/digitalocean.go

@@ -126,6 +126,28 @@ func parseInterface(iface digitalocean.Interface, nameservers []net.IP, useRoute
 			})
 		}
 	}
+	if iface.AnchorIPv4 != nil {
+		var ip, mask net.IP
+		if ip = net.ParseIP(iface.AnchorIPv4.IPAddress); ip == nil {
+			return nil, fmt.Errorf("could not parse %q as anchor IPv4 address", iface.AnchorIPv4.IPAddress)
+		}
+		if mask = net.ParseIP(iface.AnchorIPv4.Netmask); mask == nil {
+			return nil, fmt.Errorf("could not parse %q as anchor IPv4 mask", iface.AnchorIPv4.Netmask)
+		}
+		addresses = append(addresses, net.IPNet{
+			IP:   ip,
+			Mask: net.IPMask(mask),
+		})
+
+		if useRoute {
+			routes = append(routes, route{
+				destination: net.IPNet{
+					IP:   net.IPv4zero,
+					Mask: net.IPMask(net.IPv4zero),
+				},
+			})
+		}
+	}
 
 	hwaddr, err := net.ParseMAC(iface.MAC)
 	if err != nil {

network/digitalocean_test.go

@@ -258,6 +258,70 @@ func TestParseInterface(t *testing.T) {
 				},
 			},
 		},
+
+		{
+			cfg: digitalocean.Interface{
+				MAC: "01:23:45:67:89:AB",
+				AnchorIPv4: &digitalocean.Address{
+					IPAddress: "bad",
+					Netmask:   "255.255.0.0",
+				},
+			},
+			nss: []net.IP{},
+			err: errors.New("could not parse \"bad\" as anchor IPv4 address"),
+		},
+		{
+			cfg: digitalocean.Interface{
+				MAC: "01:23:45:67:89:AB",
+				AnchorIPv4: &digitalocean.Address{
+					IPAddress: "1.2.3.4",
+					Netmask:   "bad",
+				},
+			},
+			nss: []net.IP{},
+			err: errors.New("could not parse \"bad\" as anchor IPv4 mask"),
+		},
+		{
+			cfg: digitalocean.Interface{
+				MAC: "01:23:45:67:89:AB",
+				IPv4: &digitalocean.Address{
+					IPAddress: "1.2.3.4",
+					Netmask:   "255.255.0.0",
+					Gateway:   "5.6.7.8",
+				},
+				AnchorIPv4: &digitalocean.Address{
+					IPAddress: "7.8.9.10",
+					Netmask:   "255.255.0.0",
+				},
+			},
+			useRoute: true,
+			nss:      []net.IP{},
+			iface: &logicalInterface{
+				hwaddr: net.HardwareAddr([]byte{0x01, 0x23, 0x45, 0x67, 0x89, 0xab}),
+				config: configMethodStatic{
+					addresses: []net.IPNet{
+						{
+							IP:   net.ParseIP("1.2.3.4"),
+							Mask: net.IPMask(net.ParseIP("255.255.0.0")),
+						},
+						{
+							IP:   net.ParseIP("7.8.9.10"),
+							Mask: net.IPMask(net.ParseIP("255.255.0.0")),
+						},
+					},
+					nameservers: []net.IP{},
+					routes: []route{
+						{
+							destination: net.IPNet{IP: net.IPv4zero, Mask: net.IPMask(net.IPv4zero)},
+							gateway:     net.ParseIP("5.6.7.8"),
+						},
+						{
+							destination: net.IPNet{IP: net.IPv4zero, Mask: net.IPMask(net.IPv4zero)},
+						},
+					},
+				},
+			},
+		},
 	} {
 		iface, err := parseInterface(tt.cfg, tt.nss, tt.useRoute)
 		if !errorsEqual(tt.err, err) {

system/user_linux.go

@@ -18,15 +18,13 @@ import (
 	"fmt"
 	"log"
 	"os/exec"
-	"os/user"
 	"strings"
 
 	"github.com/coreos/coreos-cloudinit/config"
 )
 
 func UserExists(u *config.User) bool {
-	_, err := user.Lookup(u.Name)
-	return err == nil
+       return exec.Command("getent", "passwd", u.Name).Run() == nil
 }
 
 func CreateUser(u *config.User) error {
@@ -81,12 +79,46 @@ func CreateUser(u *config.User) error {
 	output, err := exec.Command("useradd", args...).CombinedOutput()
 	if err != nil {
 		log.Printf("Command 'useradd %s' failed: %v\n%s", strings.Join(args, " "), err, output)
+		return err
+	}
+
+	return nil
+}
+
+func IsLockedUser(u *config.User) bool {
+	output, err := exec.Command("getent", "shadow", u.Name).CombinedOutput()
+	if err == nil {
+		fields := strings.Split(string(output), ":")
+		if len(fields[1]) > 1 && fields[1][0] == '!' {
+			return true
+		}
+	}
+	return false
+}
+
+func LockUnlockUser(u *config.User) error {
+	args := []string{}
+
+	if u.LockPasswd {
+		args = append(args, "-l")
+	} else {
+		if !IsLockedUser(u) {
+			return nil
+		}
+		args = append(args, "-u")
+	}
+
+	args = append(args, u.Name)
+
+	output, err := exec.Command("passwd", args...).CombinedOutput()
+	if err != nil {
+		log.Printf("Command 'passwd %s' failed: %v\n%s", strings.Join(args, " "), err, output)
 	}
 	return err
 }
 
 func SetUserPassword(user, hash string) error {
-	cmd := exec.Command("/usr/sbin/chpasswd", "-e")
+	cmd := exec.Command("chpasswd", "-e")
 
 	stdin, err := cmd.StdinPipe()
 	if err != nil {
@@ -112,3 +144,12 @@ func SetUserPassword(user, hash string) error {
 
 	return nil
 }
+
+func UserHome(name string) (string, error) {
+	output, err := exec.Command("getent", "passwd", name).CombinedOutput()
+	if err != nil {
+		return "", err
+	}
+	passwd := strings.Split(string(output), ":")
+	return passwd[5], nil
+}

test

@@ -21,6 +21,7 @@ SRC="
 	network
 	pkg
 	system
+	.
 "
 
 echo "Checking gofix..."