feat(util/http_client): Adds generic HTTP client
Supports retries with exponential backoff as well as connection timeouts and the ability to skip SSL/TLS verification. This commit also refactors datasource and initialize packages in order to use the new HTTP client.
This commit is contained in:
Camilo Aguilar
122
util/http_client.go
Normal file
122
util/http_client.go
Normal file
@@ -0,0 +1,122 @@
|
||||
package util
|
||||
|
||||
import (
|
||||
"crypto/tls"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"log"
|
||||
"math"
|
||||
"net"
|
||||
"net/http"
|
||||
neturl "net/url"
|
||||
"strings"
|
||||
"time"
|
||||
)
|
||||
|
||||
const (
|
||||
HTTP_2xx = 2
|
||||
HTTP_4xx = 4
|
||||
)
|
||||
|
||||
type HttpClient struct {
|
||||
// Maximum exp backoff duration. Defaults to 5 seconds
|
||||
MaxBackoff time.Duration
|
||||
|
||||
// Maximum amount of connection retries. Defaults to 15
|
||||
MaxRetries int
|
||||
|
||||
// HTTP client timeout, this is suggested to be low since exponential
|
||||
// backoff will kick off too. Defaults to 2 seconds
|
||||
Timeout time.Duration
|
||||
|
||||
//Whether or not to skip TLS verification. Defaults to false
|
||||
SkipTLS bool
|
||||
}
|
||||
|
||||
func NewHttpClient() *HttpClient {
|
||||
return &HttpClient{
|
||||
MaxBackoff: time.Second * 5,
|
||||
MaxRetries: 15,
|
||||
Timeout: time.Duration(2) * time.Second,
|
||||
SkipTLS: false,
|
||||
}
|
||||
}
|
||||
|
||||
// Fetches a given URL with support for exponential backoff and maximum retries
|
||||
func (h *HttpClient) Get(rawurl string) ([]byte, error) {
|
||||
if h == nil {
|
||||
return nil, nil
|
||||
}
|
||||
|
||||
if rawurl == "" {
|
||||
return nil, errors.New("URL is empty. Skipping.")
|
||||
}
|
||||
|
||||
url, err := neturl.Parse(rawurl)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// Unfortunately, url.Parse is too generic to throw errors if a URL does not
|
||||
// have a valid HTTP scheme. So, we have to do this extra validation
|
||||
if !strings.HasPrefix(url.Scheme, "http") {
|
||||
return nil, fmt.Errorf("URL %s does not have a valid HTTP scheme. Skipping.", rawurl)
|
||||
}
|
||||
|
||||
dataURL := url.String()
|
||||
|
||||
// We need to create our own client in order to add timeout support.
|
||||
// TODO(c4milo) Replace it once Go 1.3 is officially used by CoreOS
|
||||
// More info: https://code.google.com/p/go/source/detail?r=ada6f2d5f99f
|
||||
transport := &http.Transport{
|
||||
TLSClientConfig: &tls.Config{
|
||||
InsecureSkipVerify: h.SkipTLS,
|
||||
},
|
||||
Dial: func(network, addr string) (net.Conn, error) {
|
||||
deadline := time.Now().Add(h.Timeout)
|
||||
c, err := net.DialTimeout(network, addr, h.Timeout)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
c.SetDeadline(deadline)
|
||||
return c, nil
|
||||
},
|
||||
}
|
||||
|
||||
client := &http.Client{
|
||||
Transport: transport,
|
||||
}
|
||||
|
||||
for retry := 1; retry <= h.MaxRetries; retry++ {
|
||||
log.Printf("Fetching data from %s. Attempt #%d", dataURL, retry)
|
||||
|
||||
resp, err := client.Get(dataURL)
|
||||
|
||||
if err == nil {
|
||||
defer resp.Body.Close()
|
||||
status := resp.StatusCode / 100
|
||||
|
||||
if status == HTTP_2xx {
|
||||
return ioutil.ReadAll(resp.Body)
|
||||
}
|
||||
|
||||
if status == HTTP_4xx {
|
||||
return nil, fmt.Errorf("Not found. HTTP status code: %d", resp.StatusCode)
|
||||
}
|
||||
|
||||
log.Printf("Server error. HTTP status code: %d", resp.StatusCode)
|
||||
} else {
|
||||
log.Printf("Unable to fetch data: %s", err.Error())
|
||||
}
|
||||
|
||||
duration := time.Millisecond * time.Duration((math.Pow(float64(2), float64(retry)) * 100))
|
||||
if duration > h.MaxBackoff {
|
||||
duration = h.MaxBackoff
|
||||
}
|
||||
|
||||
time.Sleep(duration)
|
||||
}
|
||||
|
||||
return nil, fmt.Errorf("Unable to fetch data. Maximum retries reached: %d", h.MaxRetries)
|
||||
}
|
||||
125
util/http_client_test.go
Normal file
125
util/http_client_test.go
Normal file
@@ -0,0 +1,125 @@
|
||||
package util
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"io"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"testing"
|
||||
)
|
||||
|
||||
var expBackoffTests = []struct {
|
||||
count int
|
||||
body string
|
||||
}{
|
||||
{0, "number of attempts: 0"},
|
||||
{1, "number of attempts: 1"},
|
||||
{2, "number of attempts: 2"},
|
||||
}
|
||||
|
||||
// Test exponential backoff and that it continues retrying if a 5xx response is
|
||||
// received
|
||||
func TestFetchURLExpBackOff(t *testing.T) {
|
||||
client := NewHttpClient()
|
||||
|
||||
for i, tt := range expBackoffTests {
|
||||
mux := http.NewServeMux()
|
||||
count := 0
|
||||
mux.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
|
||||
if count == tt.count {
|
||||
io.WriteString(w, fmt.Sprintf("number of attempts: %d", count))
|
||||
return
|
||||
}
|
||||
count++
|
||||
http.Error(w, "", 500)
|
||||
})
|
||||
ts := httptest.NewServer(mux)
|
||||
defer ts.Close()
|
||||
|
||||
data, err := client.Get(ts.URL)
|
||||
if err != nil {
|
||||
t.Errorf("Test case %d produced error: %v", i, err)
|
||||
}
|
||||
|
||||
if count != tt.count {
|
||||
t.Errorf("Test case %d failed: %d != %d", i, count, tt.count)
|
||||
}
|
||||
|
||||
if string(data) != tt.body {
|
||||
t.Errorf("Test case %d failed: %s != %s", i, tt.body, data)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Test that it stops retrying if a 4xx response comes back
|
||||
func TestFetchURL4xx(t *testing.T) {
|
||||
client := NewHttpClient()
|
||||
retries := 0
|
||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
retries++
|
||||
http.Error(w, "", 404)
|
||||
}))
|
||||
defer ts.Close()
|
||||
|
||||
_, err := client.Get(ts.URL)
|
||||
if err == nil {
|
||||
t.Errorf("Incorrect result\ngot: %s\nwant: %s", err.Error(), "Not found. HTTP status code: 404")
|
||||
}
|
||||
|
||||
if retries > 1 {
|
||||
t.Errorf("Number of retries:\n%d\nExpected number of retries:\n%s", retries, 1)
|
||||
}
|
||||
}
|
||||
|
||||
// Test that it fetches and returns user-data just fine
|
||||
func TestFetchURL2xx(t *testing.T) {
|
||||
var cloudcfg = `
|
||||
#cloud-config
|
||||
coreos:
|
||||
oem:
|
||||
id: test
|
||||
name: CoreOS.box for Test
|
||||
version-id: %VERSION_ID%+%BUILD_ID%
|
||||
home-url: https://github.com/coreos/coreos-cloudinit
|
||||
bug-report-url: https://github.com/coreos/coreos-cloudinit
|
||||
update:
|
||||
reboot-strategy: best-effort
|
||||
`
|
||||
|
||||
client := NewHttpClient()
|
||||
ts := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
fmt.Fprint(w, cloudcfg)
|
||||
}))
|
||||
defer ts.Close()
|
||||
|
||||
data, err := client.Get(ts.URL)
|
||||
if err != nil {
|
||||
t.Errorf("Incorrect result\ngot: %v\nwant: %v", err, nil)
|
||||
}
|
||||
|
||||
if string(data) != cloudcfg {
|
||||
t.Errorf("Incorrect result\ngot: %s\nwant: %s", string(data), cloudcfg)
|
||||
}
|
||||
}
|
||||
|
||||
// Test attempt to fetching using malformed URL
|
||||
func TestFetchURLMalformed(t *testing.T) {
|
||||
client := NewHttpClient()
|
||||
|
||||
var tests = []struct {
|
||||
url string
|
||||
want string
|
||||
}{
|
||||
{"boo", "URL boo does not have a valid HTTP scheme. Skipping."},
|
||||
{"mailto://boo", "URL mailto://boo does not have a valid HTTP scheme. Skipping."},
|
||||
{"ftp://boo", "URL ftp://boo does not have a valid HTTP scheme. Skipping."},
|
||||
{"", "URL is empty. Skipping."},
|
||||
}
|
||||
|
||||
for _, test := range tests {
|
||||
_, err := client.Get(test.url)
|
||||
if err == nil || err.Error() != test.want {
|
||||
t.Errorf("Incorrect result\ngot: %v\nwant: %v", err, test.want)
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user