From 31f61d75318acad96addfc47c5faf7d043f9eed8 Mon Sep 17 00:00:00 2001
From: Kelsey Hightower <kelsey.hightower@gmail.com>
Date: Wed, 14 May 2014 22:20:40 -0700
Subject: [PATCH 1/2] Use exponential backoff when fetching user-data from an
 URL.

The user-cloudinit-proc-cmdline systemd unit is responsible for
fetching user-data from various sources during the cloud-init
process. When fetching user-data from an URL datasource we face
a race condition since the network may not be available, which
can cause the job to fail and no further attempts to fetch the
user-data are made.

Eliminate the race condition when fetching user-data from an URL
datasource. Retry the fetch using an exponential backoff until
the user-data is retrieved.

Fixes issue 105.
---
 datasource/datasource.go      | 30 +++++++++++++++++------
 datasource/datasource_test.go | 45 +++++++++++++++++++++++++++++++++++
 2 files changed, 68 insertions(+), 7 deletions(-)
 create mode 100644 datasource/datasource_test.go

diff --git a/datasource/datasource.go b/datasource/datasource.go
index f6175a9..665f9ab 100644
--- a/datasource/datasource.go
+++ b/datasource/datasource.go
@@ -2,26 +2,24 @@ package datasource
 
 import (
 	"io/ioutil"
+	"log"
+	"math"
 	"net/http"
+	"time"
 )
 
 type Datasource interface {
 	Fetch() ([]byte, error)
-	Type()  string
+	Type() string
 }
 
 func fetchURL(url string) ([]byte, error) {
-	client := http.Client{}
-	resp, err := client.Get(url)
+	resp, err := getWithExponentialBackoff(url)
 	if err != nil {
 		return []byte{}, err
 	}
 	defer resp.Body.Close()
 
-	if resp.StatusCode / 100 != 2 {
-		return []byte{}, nil
-	}
-
 	respBytes, err := ioutil.ReadAll(resp.Body)
 	if err != nil {
 		return nil, err
@@ -29,3 +27,21 @@ func fetchURL(url string) ([]byte, error) {
 
 	return respBytes, nil
 }
+
+// getWithExponentialBackoff issues a GET to the specified URL. If the
+// response is a non-2xx or produces an error, retry the GET forever using
+// an exponential backoff.
+func getWithExponentialBackoff(url string) (*http.Response, error) {
+	var err error
+	var resp *http.Response
+	for i := 0; ; i++ {
+		resp, err = http.Get(url)
+		if err == nil && resp.StatusCode/100 == 2 {
+			return resp, nil
+		}
+		duration := time.Millisecond * time.Duration((math.Pow(float64(2), float64(i)) * 100))
+		log.Printf("unable to fetch user-data from %s, try again in %s", url, duration)
+		time.Sleep(duration)
+	}
+	return resp, err
+}
diff --git a/datasource/datasource_test.go b/datasource/datasource_test.go
new file mode 100644
index 0000000..19ccd53
--- /dev/null
+++ b/datasource/datasource_test.go
@@ -0,0 +1,45 @@
+package datasource
+
+import (
+	"fmt"
+	"io"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+)
+
+var expBackoffTests = []struct {
+	count int
+	body  string
+}{
+	{0, "number of attempts: 0"},
+	{1, "number of attempts: 1"},
+	{2, "number of attempts: 2"},
+}
+
+func TestGetWithExponentialBackoff(t *testing.T) {
+	for i, tt := range expBackoffTests {
+		mux := http.NewServeMux()
+		count := 0
+		mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
+			if count == tt.count {
+				io.WriteString(w, fmt.Sprintf("number of attempts: %d", count))
+				return
+			}
+			count++
+			http.Error(w, "", 500)
+		})
+		ts := httptest.NewServer(mux)
+		defer ts.Close()
+		data, err := fetchURL(ts.URL)
+		if err != nil {
+			t.Errorf("Test case %d produced error: %v", i, err)
+		}
+		if count != tt.count {
+			t.Errorf("Test case %d failed: %d != %d", i, count, tt.count)
+		}
+		if string(data) != tt.body {
+			t.Errorf("Test case %d failed: %s != %s", i, tt.body, data)
+		}
+	}
+}

From ff70a60fbc94726ac0a9217c0f9bc75bd9adba88 Mon Sep 17 00:00:00 2001
From: Camilo Aguilar <camilo.aguilar@gmail.com>
Date: Thu, 15 May 2014 13:04:37 -0400
Subject: [PATCH 2/2] Adds sleep cap to exponential backoff so it does not go
 too high

---
 datasource/datasource.go | 6 ++++++
 1 file changed, 6 insertions(+)

diff --git a/datasource/datasource.go b/datasource/datasource.go
index 665f9ab..42d9551 100644
--- a/datasource/datasource.go
+++ b/datasource/datasource.go
@@ -8,6 +8,8 @@ import (
 	"time"
 )
 
+const maxTimeout = time.Second * 5
+
 type Datasource interface {
 	Fetch() ([]byte, error)
 	Type() string
@@ -40,6 +42,10 @@ func getWithExponentialBackoff(url string) (*http.Response, error) {
 			return resp, nil
 		}
 		duration := time.Millisecond * time.Duration((math.Pow(float64(2), float64(i)) * 100))
+		if duration > maxTimeout {
+			duration = maxTimeout
+		}
+
 		log.Printf("unable to fetch user-data from %s, try again in %s", url, duration)
 		time.Sleep(duration)
 	}