feat(util/http_client): Adds generic HTTP client
Supports retries with exponential backoff as well as connection timeouts and the ability to skip SSL/TLS verification. This commit also refactors datasource and initialize packages in order to use the new HTTP client.
This commit is contained in:
Camilo Aguilar
@@ -1,104 +1,6 @@
|
||||
package datasource
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"math"
|
||||
"net"
|
||||
"net/http"
|
||||
neturl "net/url"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
HTTP_2xx = 2
|
||||
HTTP_4xx = 4
|
||||
|
||||
maxTimeout = time.Second * 5
|
||||
maxRetries = 15
|
||||
)
|
||||
|
||||
type Datasource interface {
|
||||
Fetch() ([]byte, error)
|
||||
Type() string
|
||||
}
|
||||
|
||||
// HTTP client timeout
|
||||
// This one is low since exponential backoff will kick off too.
|
||||
var timeout = time.Duration(2) * time.Second
|
||||
|
||||
func dialTimeout(network, addr string) (net.Conn, error) {
|
||||
deadline := time.Now().Add(timeout)
|
||||
c, err := net.DialTimeout(network, addr, timeout)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
c.SetDeadline(deadline)
|
||||
return c, nil
|
||||
}
|
||||
|
||||
// Fetches user-data url with support for exponential backoff and maximum retries
|
||||
func fetchURL(rawurl string) ([]byte, error) {
|
||||
if rawurl == "" {
|
||||
return nil, errors.New("user-data URL is empty. Skipping.")
|
||||
}
|
||||
|
||||
url, err := neturl.Parse(rawurl)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Unfortunately, url.Parse is too generic to throw errors if a URL does not
|
||||
// have a valid HTTP scheme. So, we have to do this extra validation
|
||||
if !strings.HasPrefix(url.Scheme, "http") {
|
||||
return nil, fmt.Errorf("user-data URL %s does not have a valid HTTP scheme. Skipping.", rawurl)
|
||||
}
|
||||
|
||||
userdataURL := url.String()
|
||||
|
||||
// We need to create our own client in order to add timeout support.
|
||||
// TODO(c4milo) Replace it once Go 1.3 is officially used by CoreOS
|
||||
// More info: https://code.google.com/p/go/source/detail?r=ada6f2d5f99f
|
||||
transport := &http.Transport{
|
||||
Dial: dialTimeout,
|
||||
}
|
||||
|
||||
client := &http.Client{
|
||||
Transport: transport,
|
||||
}
|
||||
|
||||
for retry := 1; retry <= maxRetries; retry++ {
|
||||
log.Printf("Fetching user-data from %s. Attempt #%d", userdataURL, retry)
|
||||
|
||||
resp, err := client.Get(userdataURL)
|
||||
|
||||
if err == nil {
|
||||
defer resp.Body.Close()
|
||||
status := resp.StatusCode / 100
|
||||
|
||||
if status == HTTP_2xx {
|
||||
return ioutil.ReadAll(resp.Body)
|
||||
}
|
||||
|
||||
if status == HTTP_4xx {
|
||||
return nil, fmt.Errorf("user-data not found. HTTP status code: %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
log.Printf("user-data not found. HTTP status code: %d", resp.StatusCode)
|
||||
} else {
|
||||
log.Printf("unable to fetch user-data: %s", err.Error())
|
||||
}
|
||||
|
||||
duration := time.Millisecond * time.Duration((math.Pow(float64(2), float64(retry)) * 100))
|
||||
if duration > maxTimeout {
|
||||
duration = maxTimeout
|
||||
}
|
||||
|
||||
time.Sleep(duration)
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("unable to fetch user-data. Maximum retries reached: %d", maxRetries)
|
||||
}
|
||||
|
||||
@@ -1,119 +0,0 @@
|
||||
package datasource
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
)
|
||||
|
||||
var expBackoffTests = []struct {
|
||||
count int
|
||||
body string
|
||||
}{
|
||||
{0, "number of attempts: 0"},
|
||||
{1, "number of attempts: 1"},
|
||||
{2, "number of attempts: 2"},
|
||||
}
|
||||
|
||||
// Test exponential backoff and that it continues retrying if a 5xx response is
|
||||
// received
|
||||
func TestFetchURLExpBackOff(t *testing.T) {
|
||||
for i, tt := range expBackoffTests {
|
||||
mux := http.NewServeMux()
|
||||
count := 0
|
||||
mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
|
||||
if count == tt.count {
|
||||
io.WriteString(w, fmt.Sprintf("number of attempts: %d", count))
|
||||
return
|
||||
}
|
||||
count++
|
||||
http.Error(w, "", 500)
|
||||
})
|
||||
ts := httptest.NewServer(mux)
|
||||
defer ts.Close()
|
||||
|
||||
data, err := fetchURL(ts.URL)
|
||||
if err != nil {
|
||||
t.Errorf("Test case %d produced error: %v", i, err)
|
||||
}
|
||||
|
||||
if count != tt.count {
|
||||
t.Errorf("Test case %d failed: %d != %d", i, count, tt.count)
|
||||
}
|
||||
|
||||
if string(data) != tt.body {
|
||||
t.Errorf("Test case %d failed: %s != %s", i, tt.body, data)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Test that it stops retrying if a 4xx response comes back
|
||||
func TestFetchURL4xx(t *testing.T) {
|
||||
retries := 0
|
||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
retries++
|
||||
http.Error(w, "", 404)
|
||||
}))
|
||||
defer ts.Close()
|
||||
|
||||
_, err := fetchURL(ts.URL)
|
||||
if err == nil {
|
||||
t.Errorf("Incorrect result\ngot: %s\nwant: %s", err.Error(), "user-data not found. HTTP status code: 404")
|
||||
}
|
||||
|
||||
if retries > 1 {
|
||||
t.Errorf("Number of retries:\n%d\nExpected number of retries:\n%s", retries, 1)
|
||||
}
|
||||
}
|
||||
|
||||
// Test that it fetches and returns user-data just fine
|
||||
func TestFetchURL2xx(t *testing.T) {
|
||||
var cloudcfg = `
|
||||
#cloud-config
|
||||
coreos:
|
||||
oem:
|
||||
id: test
|
||||
name: CoreOS.box for Test
|
||||
version-id: %VERSION_ID%+%BUILD_ID%
|
||||
home-url: https://github.com/coreos/coreos-cloudinit
|
||||
bug-report-url: https://github.com/coreos/coreos-cloudinit
|
||||
update:
|
||||
reboot-strategy: best-effort
|
||||
`
|
||||
|
||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
fmt.Fprint(w, cloudcfg)
|
||||
}))
|
||||
defer ts.Close()
|
||||
|
||||
data, err := fetchURL(ts.URL)
|
||||
if err != nil {
|
||||
t.Errorf("Incorrect result\ngot: %v\nwant: %v", err, nil)
|
||||
}
|
||||
|
||||
if string(data) != cloudcfg {
|
||||
t.Errorf("Incorrect result\ngot: %s\nwant: %s", string(data), cloudcfg)
|
||||
}
|
||||
}
|
||||
|
||||
// Test attempt to fetching using malformed URL
|
||||
func TestFetchURLMalformed(t *testing.T) {
|
||||
var tests = []struct {
|
||||
url string
|
||||
want string
|
||||
}{
|
||||
{"boo", "user-data URL boo does not have a valid HTTP scheme. Skipping."},
|
||||
{"mailto://boo", "user-data URL mailto://boo does not have a valid HTTP scheme. Skipping."},
|
||||
{"ftp://boo", "user-data URL ftp://boo does not have a valid HTTP scheme. Skipping."},
|
||||
{"", "user-data URL is empty. Skipping."},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
_, err := fetchURL(test.url)
|
||||
if err == nil || err.Error() != test.want {
|
||||
t.Errorf("Incorrect result\ngot: %v\nwant: %v", err, test.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -1,5 +1,7 @@
|
||||
package datasource
|
||||
|
||||
import "github.com/coreos/coreos-cloudinit/util"
|
||||
|
||||
type metadataService struct {
|
||||
url string
|
||||
}
|
||||
@@ -9,7 +11,8 @@ func NewMetadataService(url string) *metadataService {
|
||||
}
|
||||
|
||||
func (ms *metadataService) Fetch() ([]byte, error) {
|
||||
return fetchURL(ms.url)
|
||||
client := util.NewHttpClient()
|
||||
return client.Get(ms.url)
|
||||
}
|
||||
|
||||
func (ms *metadataService) Type() string {
|
||||
|
||||
@@ -5,6 +5,8 @@ import (
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"strings"
|
||||
|
||||
"github.com/coreos/coreos-cloudinit/util"
|
||||
)
|
||||
|
||||
const (
|
||||
@@ -29,7 +31,8 @@ func (self *procCmdline) Fetch() ([]byte, error) {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
cfg, err := fetchURL(url)
|
||||
client := util.NewHttpClient()
|
||||
cfg, err := client.Get(url)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
Reference in New Issue
Block a user