cloudinit/pkg/http_client.go

/*
   Copyright 2014 CoreOS, Inc.

   Licensed under the Apache License, Version 2.0 (the "License");
   you may not use this file except in compliance with the License.
   You may obtain a copy of the License at

       http://www.apache.org/licenses/LICENSE-2.0

   Unless required by applicable law or agreed to in writing, software
   distributed under the License is distributed on an "AS IS" BASIS,
   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
   See the License for the specific language governing permissions and
   limitations under the License.
*/

package pkg

import (
	"crypto/tls"
	"errors"
	"fmt"
	"io/ioutil"
	"log"
	"net"
	"net/http"
	neturl "net/url"
	"strings"
	"time"
)

const (
	HTTP_2xx = 2
	HTTP_4xx = 4
)

type Err error

type ErrTimeout struct {
	Err
}

type ErrNotFound struct {
	Err
}

type ErrInvalid struct {
	Err
}

type ErrServer struct {
	Err
}

type ErrNetwork struct {
	Err
}

type HttpClient struct {
	// Maximum exp backoff duration. Defaults to 5 seconds
	MaxBackoff time.Duration

	// Maximum number of connection retries. Defaults to 15
	MaxRetries int

	// HTTP client timeout, this is suggested to be low since exponential
	// backoff will kick off too. Defaults to 2 seconds
	Timeout time.Duration

	// Whether or not to skip TLS verification. Defaults to false
	SkipTLS bool

	client *http.Client
}

type Getter interface {
	Get(string) ([]byte, error)
	GetRetry(string) ([]byte, error)
}

func NewHttpClient() *HttpClient {
	hc := &HttpClient{
		MaxBackoff: time.Second * 5,
		MaxRetries: 15,
		Timeout:    time.Duration(2) * time.Second,
		SkipTLS:    false,
	}

	// We need to create our own client in order to add timeout support.
	// TODO(c4milo) Replace it once Go 1.3 is officially used by CoreOS
	// More info: https://code.google.com/p/go/source/detail?r=ada6f2d5f99f
	hc.client = &http.Client{
		Transport: &http.Transport{
			TLSClientConfig: &tls.Config{
				InsecureSkipVerify: hc.SkipTLS,
			},
			Dial: func(network, addr string) (net.Conn, error) {
				deadline := time.Now().Add(hc.Timeout)
				c, err := net.DialTimeout(network, addr, hc.Timeout)
				if err != nil {
					return nil, err
				}
				c.SetDeadline(deadline)
				return c, nil
			},
		},
	}

	return hc
}

func ExpBackoff(interval, max time.Duration) time.Duration {
	interval = interval * 2
	if interval > max {
		interval = max
	}
	return interval
}

// GetRetry fetches a given URL with support for exponential backoff and maximum retries
func (h *HttpClient) GetRetry(rawurl string) ([]byte, error) {
	if rawurl == "" {
		return nil, ErrInvalid{errors.New("URL is empty. Skipping.")}
	}

	url, err := neturl.Parse(rawurl)
	if err != nil {
		return nil, ErrInvalid{err}
	}

	// Unfortunately, url.Parse is too generic to throw errors if a URL does not
	// have a valid HTTP scheme. So, we have to do this extra validation
	if !strings.HasPrefix(url.Scheme, "http") {
		return nil, ErrInvalid{fmt.Errorf("URL %s does not have a valid HTTP scheme. Skipping.", rawurl)}
	}

	dataURL := url.String()

	duration := 50 * time.Millisecond
	for retry := 1; retry <= h.MaxRetries; retry++ {
		log.Printf("Fetching data from %s. Attempt #%d", dataURL, retry)

		data, err := h.Get(dataURL)
		switch err.(type) {
		case ErrNetwork:
			log.Printf(err.Error())
		case ErrServer:
			log.Printf(err.Error())
		case ErrNotFound:
			return data, err
		default:
			return data, err
		}

		duration = ExpBackoff(duration, h.MaxBackoff)
		log.Printf("Sleeping for %v...", duration)
		time.Sleep(duration)
	}

	return nil, ErrTimeout{fmt.Errorf("Unable to fetch data. Maximum retries reached: %d", h.MaxRetries)}
}

func (h *HttpClient) Get(dataURL string) ([]byte, error) {
	if resp, err := h.client.Get(dataURL); err == nil {
		defer resp.Body.Close()
		switch resp.StatusCode / 100 {
		case HTTP_2xx:
			return ioutil.ReadAll(resp.Body)
		case HTTP_4xx:
			return nil, ErrNotFound{fmt.Errorf("Not found. HTTP status code: %d", resp.StatusCode)}
		default:
			return nil, ErrServer{fmt.Errorf("Server error. HTTP status code: %d", resp.StatusCode)}
		}
	} else {
		return nil, ErrNetwork{fmt.Errorf("Unable to fetch data: %s", err.Error())}
	}
}