Tune HTTP client for concurrency

- due to what appears to be a frequent issue with the Go HTTP
client some tweaks were needed to the HTTP client used for
reverse proxying to prevent CoreDNS from rejecting connections.

The following PRs / commits implement similar changes in
Prometheus and Minio.

https://github.com/prometheus/prometheus/pull/3592
https://github.com/minio/minio/pull/5860

Under a 3-node (1-master) kubeadm cluster running on bare
metal with Ubuntu 18.04 I was able to send 100k requests
with 1000 being concurrent with no errors being returned
by hey.

```
hey -n 100000 -c 1000 -m=POST -d="hi" \
  http://192.168.0.26:31112/function/go-echo
```

The go-echo function is based upon the golang-http
template in the function store using the of-watchdog.

Signed-off-by: Alex Ellis (VMware) <alexellis2@gmail.com>

gateway/handlers/forwarding_proxy.go

@@ -40,6 +40,12 @@ type URLPathTransformer interface {
 
 // MakeForwardingProxyHandler create a handler which forwards HTTP requests
 func MakeForwardingProxyHandler(proxy *types.HTTPClientReverseProxy, notifiers []HTTPNotifier, baseURLResolver BaseURLResolver, urlPathTransformer URLPathTransformer) http.HandlerFunc {
+
+	writeRequestURI := false
+	if _, exists := os.LookupEnv("write_request_uri"); exists {
+		writeRequestURI = exists
+	}
+
 	return func(w http.ResponseWriter, r *http.Request) {
 		baseURL := baseURLResolver.Resolve(r)
 		originalURL := r.URL.String()
@@ -48,16 +54,19 @@ func MakeForwardingProxyHandler(proxy *types.HTTPClientReverseProxy, notifiers [
 
 		start := time.Now()
 
-		statusCode, err := forwardRequest(w, r, proxy.Client, baseURL, requestURL, proxy.Timeout)
+		statusCode, err := forwardRequest(w, r, proxy.Client, baseURL, requestURL, proxy.Timeout, writeRequestURI)
 
 		seconds := time.Since(start)
 		if err != nil {
 			log.Printf("error with upstream request to: %s, %s\n", requestURL, err.Error())
 		}
 
+		defer func() {
 			for _, notifier := range notifiers {
 				notifier.Notify(r.Method, requestURL, originalURL, statusCode, seconds)
 			}
+		}()
+
 	}
 }
 
@@ -86,14 +95,14 @@ func buildUpstreamRequest(r *http.Request, baseURL string, requestURL string) *h
 	return upstreamReq
 }
 
-func forwardRequest(w http.ResponseWriter, r *http.Request, proxyClient *http.Client, baseURL string, requestURL string, timeout time.Duration) (int, error) {
+func forwardRequest(w http.ResponseWriter, r *http.Request, proxyClient *http.Client, baseURL string, requestURL string, timeout time.Duration, writeRequestURI bool) (int, error) {
 
 	upstreamReq := buildUpstreamRequest(r, baseURL, requestURL)
 	if upstreamReq.Body != nil {
 		defer upstreamReq.Body.Close()
 	}
 
-	if _, exists := os.LookupEnv("write_request_uri"); exists {
+	if writeRequestURI {
 		log.Printf("forwardRequest: %s %s\n", upstreamReq.Host, upstreamReq.URL.String())
 	}
 

gateway/types/proxy_client.go

@@ -17,20 +17,25 @@ func NewHTTPClientReverseProxy(baseURL *url.URL, timeout time.Duration) *HTTPCli
 		Timeout: timeout,
 	}
 
-	h.Client = &http.Client{
+	h.Client = http.DefaultClient
-		Transport: &http.Transport{
+	h.Timeout = timeout
+	h.Client.CheckRedirect = func(req *http.Request, via []*http.Request) error {
+		return http.ErrUseLastResponse
+	}
+
+	// Taken from http.DefaultTransport in Go 1.11
+	h.Client.Transport = &http.Transport{
 		Proxy: http.ProxyFromEnvironment,
 		DialContext: (&net.Dialer{
 			Timeout:   timeout,
-				KeepAlive: 1 * time.Second,
+			KeepAlive: timeout,
+			DualStack: true,
 		}).DialContext,
-			IdleConnTimeout:       120 * time.Millisecond,
+		MaxIdleConns:          20000, // Overriden via https://github.com/errordeveloper/prometheus/commit/1f74477646aea93bebb7c098affa8e05132abb0c
-			ExpectContinueTimeout: 1500 * time.Millisecond,
+		MaxIdleConnsPerHost:   1024,  // Overriden via https://github.com/minio/minio/pull/5860
-		},
+		IdleConnTimeout:       90 * time.Second,
-		Timeout: timeout,
+		TLSHandshakeTimeout:   10 * time.Second,
-		CheckRedirect: func(req *http.Request, via []*http.Request) error {
+		ExpectContinueTimeout: 1 * time.Second,
-			return http.ErrUseLastResponse
-		},
 	}
 
 	return &h