Verify not goroutine leaks in the log proxy

**What**
- Add test to verify that the log proxy shutsdown correctly when the
client cancels
- Add test to verify that the log proxy shutsdown correctly when the
logs provider closes the connection

Signed-off-by: Lucas Roesler <roesler.lucas@gmail.com>

gateway/Gopkg.lock

@@ -154,6 +154,17 @@
   pruneopts = ""
   revision = "b15cd069a83443be3154b719d0cc9fe8117f09fb"
 
+[[projects]]
+  digest = "1:60bdcc3a64276ffef2e43430704d6b2488e3fd934475149e932e220251dbe131"
+  name = "go.uber.org/goleak"
+  packages = [
+    ".",
+    "internal/stack",
+  ]
+  pruneopts = ""
+  revision = "1ac8aeca0a53163331564467638f6ffb639636bf"
+  version = "v0.10.0"
+
 [solve-meta]
   analyzer-name = "dep"
   analyzer-version = 1
@@ -165,6 +176,7 @@
     "github.com/prometheus/client_golang/prometheus",
     "github.com/prometheus/client_golang/prometheus/promhttp",
     "github.com/prometheus/client_model/go",
+    "go.uber.org/goleak",
   ]
   solver-name = "gps-cdcl"
   solver-version = 1

gateway/Gopkg.toml

@@ -27,3 +27,7 @@ ignored = ["github.com/openfaas/faas/gateway/queue"]
 [[constraint]]
   name = "github.com/openfaas/faas-provider"
   version = "0.9.1"
+
+[[constraint]]
+  name = "go.uber.org/goleak"
+  version = "0.10.0"

gateway/handlers/logs_test.go

@@ -0,0 +1,133 @@
+package handlers
+
+import (
+	"context"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
+	"net/url"
+	"testing"
+
+	"go.uber.org/goleak"
+)
+
+func Test_logsProxyDoesNotLeakGoroutinesWhenProviderClosesConnection(t *testing.T) {
+	defer goleak.VerifyNoLeaks(t)
+
+	expectedMsg := "name: funcFoo msg: test message"
+
+	// mock log provider that sends one line and immediately closes the connection
+	mockLogsUpstreamEndpoint := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		if r.Method != http.MethodGet {
+			t.Fatalf("expected method '%s' but got '%s'", http.MethodGet, r.Method)
+		}
+
+		if r.URL.Path != upstreamLogsEndpoint {
+			t.Fatalf("expected path '%s' but got '%s'", upstreamLogsEndpoint, r.URL.Path)
+		}
+
+		w.Header().Set(http.CanonicalHeaderKey("Connection"), "Keep-Alive")
+		w.Header().Set(http.CanonicalHeaderKey("Transfer-Encoding"), "chunked")
+		w.Header().Set(http.CanonicalHeaderKey("Content-Type"), "application/x-ndjson")
+		w.WriteHeader(http.StatusOK)
+
+		msg := fmt.Sprintf("name: %s msg: test message", r.URL.Query().Get("name"))
+		_, err := w.Write([]byte(msg))
+		if err != nil {
+			t.Fatalf("failed to write test log message: %s", err)
+		}
+	}))
+	defer mockLogsUpstreamEndpoint.Close()
+
+	logProviderURL, _ := url.Parse(mockLogsUpstreamEndpoint.URL)
+
+	logHandler := NewLogHandlerFunc(*logProviderURL)
+	testSrv := httptest.NewServer(http.HandlerFunc(logHandler))
+	defer testSrv.Close()
+
+	resp, err := http.Get(testSrv.URL + "?name=funcFoo")
+	if err != nil {
+		t.Fatalf("unexpected error sneding log request: %s", err)
+	}
+
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		t.Fatalf("unexpected error reading teh response body: %s", err)
+	}
+
+	if string(body) != string(expectedMsg) {
+		t.Fatalf("expected log message %s, got: %s", expectedMsg, body)
+	}
+}
+
+func Test_logsProxyDoesNotLeakGoroutinesWhenClientClosesConnection(t *testing.T) {
+	defer goleak.VerifyNoLeaks(t)
+
+	// mock log provider that sends one line and holds until we cancel the context
+	mockLogsUpstreamEndpoint := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
+		cn, ok := w.(http.CloseNotifier)
+		if !ok {
+			http.Error(w, "cannot stream", http.StatusInternalServerError)
+			return
+		}
+
+		flusher, ok := w.(http.Flusher)
+		if !ok {
+			http.Error(w, "cannot stream", http.StatusInternalServerError)
+			return
+		}
+
+		if r.Method != http.MethodGet {
+			t.Fatalf("expected method '%s' but got '%s'", http.MethodGet, r.Method)
+		}
+
+		if r.URL.Path != upstreamLogsEndpoint {
+			t.Fatalf("expected path '%s' but got '%s'", upstreamLogsEndpoint, r.URL.Path)
+		}
+
+		w.Header().Set(http.CanonicalHeaderKey("Connection"), "Keep-Alive")
+		w.Header().Set(http.CanonicalHeaderKey("Transfer-Encoding"), "chunked")
+		w.Header().Set(http.CanonicalHeaderKey("Content-Type"), "application/x-ndjson")
+		w.WriteHeader(http.StatusOK)
+
+		msg := fmt.Sprintf("name: %s msg: test message", r.URL.Query().Get("name"))
+		_, err := w.Write([]byte(msg))
+		if err != nil {
+			t.Fatalf("failed to write test log message: %s", err)
+		}
+
+		flusher.Flush()
+
+		// "wait for connection to close"
+		<-cn.CloseNotify()
+
+	}))
+	defer mockLogsUpstreamEndpoint.Close()
+
+	logProviderURL, _ := url.Parse(mockLogsUpstreamEndpoint.URL)
+
+	logHandler := NewLogHandlerFunc(*logProviderURL)
+	testSrv := httptest.NewServer(http.HandlerFunc(logHandler))
+	defer testSrv.Close()
+
+	reqContext, cancel := context.WithCancel(context.Background())
+	req, _ := http.NewRequest(http.MethodGet, testSrv.URL+"?name=funcFoo", nil)
+
+	req = req.WithContext(reqContext)
+
+	resp, err := http.DefaultClient.Do(req)
+	if err != nil {
+		t.Fatalf("unexpected error sending log request: %s", err)
+	}
+
+	go func() {
+		defer resp.Body.Close()
+		_, err := ioutil.ReadAll(resp.Body)
+		if err != context.Canceled {
+			t.Fatalf("unexpected error reading the response body: %s", err)
+		}
+	}()
+
+	cancel()
+}

gateway/vendor/go.uber.org/goleak/.gitignore

@@ -0,0 +1 @@
+vendor/

gateway/vendor/go.uber.org/goleak/README.md

@@ -0,0 +1,47 @@
+# goleak [![GoDoc][doc-img]][doc] [![Build Status][ci-img]][ci] [![Coverage Status][cov-img]][cov]
+
+Goroutine leak detector to help avoid Goroutine leaks.
+
+## Development Status: Alpha
+
+goleak is still in development, and APIs are still in flux.
+
+## Installation
+
+You can use `go get` to get the latest version:
+
+`go get -u go.uber.org/goleak`
+
+`goleak` also supports semver releases. It is compatible with Go 1.5+.
+
+## Quick Start
+
+To verify that there are no unexpected goroutines running at the end of a test:
+
+```go
+func TestA(t *testing.T) {
+	defer goleak.Verify(t)
+
+	// test logic here.
+}
+```
+
+Instead of checking for leaks at the end of every test, `goleak` can also be run
+at the end of every test package by creating a `TestMain` function for your 
+package:
+
+```go
+func TestMain(m *testing.M) {
+	goleak.VerifyTestMain(m)
+}
+```
+
+
+[doc-img]: https://godoc.org/go.uber.org/goleak?status.svg
+[doc]: https://godoc.org/go.uber.org/goleak
+[ci-img]: https://travis-ci.org/uber-go/goleak.svg?branch=master
+[ci]: https://travis-ci.org/uber-go/goleak
+[cov-img]: https://codecov.io/gh/uber-go/goleak/branch/master/graph/badge.svg
+[cov]: https://codecov.io/gh/uber-go/goleak
+[benchmarking suite]: https://github.com/uber-go/goleak/tree/master/benchmarks
+[glide.lock]: https://github.com/uber-go/goleak/blob/master/glide.lock

gateway/vendor/go.uber.org/goleak/glide.lock

@@ -0,0 +1,17 @@
+hash: b4576f8060ebfcac0fa8f64e01324a30233a4e49dde3724ada74b9f055a48f91
+updated: 2017-11-14T08:37:48.249991169-08:00
+imports: []
+testImports:
+- name: github.com/davecgh/go-spew
+  version: 6d212800a42e8ab5c146b8ace3490ee17e5225f9
+  subpackages:
+  - spew
+- name: github.com/pmezard/go-difflib
+  version: d8ed2627bdf02c080bf22230dbb337003b7aba2d
+  subpackages:
+  - difflib
+- name: github.com/stretchr/testify
+  version: 69483b4bd14f5845b5a1e55bca19e954e827f1d0
+  subpackages:
+  - assert
+  - require

gateway/vendor/go.uber.org/goleak/glide.yaml

@@ -0,0 +1,8 @@
+package: go.uber.org/goleak
+import: []
+testImport:
+- package: github.com/stretchr/testify
+  version: ^1.1.4
+  subpackages:
+  - assert
+  - require

gateway/vendor/go.uber.org/goleak/internal/stack/stacks.go

@@ -0,0 +1,155 @@
+// Copyright (c) 2017 Uber Technologies, Inc.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+package stack
+
+import (
+	"bufio"
+	"bytes"
+	"fmt"
+	"io"
+	"runtime"
+	"strconv"
+	"strings"
+)
+
+const _defaultBufferSize = 64 * 1024 // 64 KiB
+
+// Stack represents a single Goroutine's stack.
+type Stack struct {
+	id            int
+	state         string
+	firstFunction string
+	fullStack     *bytes.Buffer
+}
+
+// ID returns the goroutine ID.
+func (s Stack) ID() int {
+	return s.id
+}
+
+// State returns the Goroutine's state.
+func (s Stack) State() string {
+	return s.state
+}
+
+// Full returns the full stack trace for this goroutine.
+func (s Stack) Full() string {
+	return s.fullStack.String()
+}
+
+// FirstFunction returns the name of the first function on the stack.
+func (s Stack) FirstFunction() string {
+	return s.firstFunction
+}
+
+func (s Stack) String() string {
+	return fmt.Sprintf(
+		"Goroutine %v in state %v, with %v on top of the stack:\n%s",
+		s.id, s.state, s.firstFunction, s.Full())
+}
+
+func getStacks(all bool) []Stack {
+	var stacks []Stack
+
+	var curStack *Stack
+	stackReader := bufio.NewReader(bytes.NewReader(getStackBuffer(all)))
+	for {
+		line, err := stackReader.ReadString('\n')
+		if err == io.EOF {
+			break
+		}
+		if err != nil {
+			// We're reading using bytes.NewReader which should never fail.
+			panic("bufio.NewReader failed on a fixed string")
+		}
+
+		// If we see the goroutine header, start a new stack.
+		isFirstLine := false
+		if strings.HasPrefix(line, "goroutine ") {
+			// flush any previous stack
+			if curStack != nil {
+				stacks = append(stacks, *curStack)
+			}
+			id, goState := parseGoStackHeader(line)
+			curStack = &Stack{
+				id:        id,
+				state:     goState,
+				fullStack: &bytes.Buffer{},
+			}
+			isFirstLine = true
+		}
+		curStack.fullStack.WriteString(line)
+		if !isFirstLine && curStack.firstFunction == "" {
+			curStack.firstFunction = parseFirstFunc(line)
+		}
+	}
+
+	if curStack != nil {
+		stacks = append(stacks, *curStack)
+	}
+	return stacks
+}
+
+// All returns the stacks for all running goroutines.
+func All() []Stack {
+	return getStacks(true)
+}
+
+// Current returns the stack for the current goroutine.
+func Current() Stack {
+	return getStacks(false)[0]
+}
+
+func getStackBuffer(all bool) []byte {
+	for i := _defaultBufferSize; ; i *= 2 {
+		buf := make([]byte, i)
+		if n := runtime.Stack(buf, all); n < i {
+			return buf[:n]
+		}
+	}
+}
+
+func parseFirstFunc(line string) string {
+	line = strings.TrimSpace(line)
+	if idx := strings.LastIndex(line, "("); idx > 0 {
+		return line[:idx]
+	}
+	panic(fmt.Sprintf("function calls missing parents: %q", line))
+}
+
+// parseGoStackHeader parses a stack header that looks like:
+// goroutine 643 [runnable]:\n
+// And returns the goroutine ID, and the state.
+func parseGoStackHeader(line string) (goroutineID int, state string) {
+	line = strings.TrimSuffix(line, ":\n")
+	parts := strings.SplitN(line, " ", 3)
+	if len(parts) != 3 {
+		panic(fmt.Sprintf("unexpected stack header format: %q", line))
+	}
+
+	id, err := strconv.Atoi(parts[1])
+	if err != nil {
+		panic(fmt.Sprintf("failed to parse goroutine ID: %v in line %q", parts[1], line))
+	}
+
+	state = strings.TrimSuffix(strings.TrimPrefix(parts[2], "["), "]")
+	return id, state
+}

gateway/vendor/go.uber.org/goleak/internal/stack/stacks_test.go

@@ -0,0 +1,128 @@
+// Copyright (c) 2017 Uber Technologies, Inc.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+package stack
+
+import (
+	"sort"
+	"sync"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+var _allDone chan struct{}
+
+func waitForDone() {
+	<-_allDone
+}
+
+func TestAll(t *testing.T) {
+	// We use a global channel so that the function below does not
+	// recieve any arguments, so we can test that parseFirstFunc works
+	// regardless of arguments on the stack.
+	_allDone = make(chan struct{})
+	defer close(_allDone)
+
+	for i := 0; i < 5; i++ {
+		go waitForDone()
+	}
+
+	got := All()
+
+	// We have exactly 7 gorotuines:
+	// "main" goroutine
+	// test goroutine
+	// 5 goroutines started above.
+	require.Len(t, got, 7)
+	sort.Sort(byGoroutineID(got))
+
+	assert.Contains(t, got[0].Full(), "testing.(*T).Run")
+	assert.Contains(t, got[1].Full(), "TestAll")
+	for i := 0; i < 5; i++ {
+		assert.Contains(t, got[2+i].Full(), "stack.waitForDone")
+	}
+}
+
+func TestCurrent(t *testing.T) {
+	got := Current()
+	assert.NotZero(t, got.ID(), "Should get non-zero goroutine id")
+	assert.Equal(t, "running", got.State())
+	assert.Equal(t, "go.uber.org/goleak/internal/stack.getStackBuffer", got.FirstFunction())
+
+	wantFrames := []string{
+		"stack.getStackBuffer",
+		"stack.getStacks",
+		"stack.Current",
+		"stack.Current",
+		"stack.TestCurrent",
+	}
+	all := got.Full()
+	for _, frame := range wantFrames {
+		assert.Contains(t, all, frame)
+	}
+	assert.Contains(t, got.String(), "in state")
+	assert.Contains(t, got.String(), "on top of the stack")
+
+	// Ensure that we are not returning the buffer without slicing it
+	// from getStackBuffer.
+	if len(got.Full()) > 1024 {
+		t.Fatalf("Returned stack is too large")
+	}
+}
+
+func TestAllLargeStack(t *testing.T) {
+	const (
+		stackDepth    = 100
+		numGoroutines = 100
+	)
+
+	var started sync.WaitGroup
+
+	done := make(chan struct{})
+	for i := 0; i < numGoroutines; i++ {
+		var f func(int)
+		f = func(count int) {
+			if count == 0 {
+				started.Done()
+				<-done
+			}
+			f(count - 1)
+		}
+		started.Add(1)
+		go f(stackDepth)
+	}
+
+	started.Wait()
+	buf := getStackBuffer(true /* all */)
+	if len(buf) <= _defaultBufferSize {
+		t.Fatalf("Expected larger stack buffer")
+	}
+
+	// Start enough goroutines so we exceed the default buffer size.
+	close(done)
+}
+
+type byGoroutineID []Stack
+
+func (ss byGoroutineID) Len() int           { return len(ss) }
+func (ss byGoroutineID) Less(i, j int) bool { return ss[i].ID() < ss[j].ID() }
+func (ss byGoroutineID) Swap(i, j int)      { ss[i], ss[j] = ss[j], ss[i] }

gateway/vendor/go.uber.org/goleak/leaks.go

@@ -0,0 +1,80 @@
+// Copyright (c) 2017 Uber Technologies, Inc.
+
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+package goleak
+
+import (
+	"fmt"
+
+	"go.uber.org/goleak/internal/stack"
+)
+
+// TestingT is the minimal subset of testing.TB that we use.
+type TestingT interface {
+	Error(...interface{})
+}
+
+// filterStacks will filter any stacks excluded by the given opts.
+// filterStacks modifies the passed in stacks slice.
+func filterStacks(stacks []stack.Stack, skipID int, opts *opts) []stack.Stack {
+	filtered := stacks[:0]
+	for _, stack := range stacks {
+		// Always skip the running goroutine.
+		if stack.ID() == skipID {
+			continue
+		}
+		// Run any default or user-specified filters.
+		if opts.filter(stack) {
+			continue
+		}
+		filtered = append(filtered, stack)
+	}
+	return filtered
+}
+
+// FindLeaks looks for extra goroutines, and returns a descriptive error if
+// any are found.
+func FindLeaks(options ...Option) error {
+	cur := stack.Current().ID()
+
+	opts := buildOpts(options...)
+	var stacks []stack.Stack
+	retry := true
+	for i := 0; retry; i++ {
+		stacks = filterStacks(stack.All(), cur, opts)
+
+		if len(stacks) == 0 {
+			return nil
+		}
+		retry = opts.retry(i)
+	}
+
+	return fmt.Errorf("found unexpected goroutines:\n%s", stacks)
+}
+
+// VerifyNoLeaks calls FindLeaks and calls Error on the passed in TestingT if
+// any leaks are found. This is a helper method to make it easier to integrate
+// in tests by doing:
+// defer VerifyNoLeaks(t)
+func VerifyNoLeaks(t TestingT, options ...Option) {
+	if err := FindLeaks(options...); err != nil {
+		t.Error(err)
+	}
+}

gateway/vendor/go.uber.org/goleak/leaks_test.go

@@ -0,0 +1,84 @@
+// Copyright (c) 2017 Uber Technologies, Inc.
+
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+package goleak
+
+import (
+	"fmt"
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+// Ensure that testingT is a subset of testing.TB.
+var _ = TestingT(testing.TB(nil))
+
+// testOptions passes a shorter max sleep time, used so tests don't wait
+// ~1 second in cases where we expect FindLeaks to error out.
+func testOptions() Option {
+	return maxSleep(time.Millisecond)
+}
+
+func TestFindLeaks(t *testing.T) {
+	require.NoError(t, FindLeaks(), "Should find no leaks by default")
+
+	bg := startBlockedG()
+	err := FindLeaks(testOptions())
+	require.Error(t, err, "Should find leaks with leaked goroutine")
+	assert.Contains(t, err.Error(), "blockedG")
+	assert.Contains(t, err.Error(), "created by go.uber.org/goleak.startBlockedG")
+
+	// Once we unblock the goroutine, we shouldn't have leaks.
+	bg.unblock()
+	require.NoError(t, FindLeaks(), "Should find no leaks by default")
+}
+
+func TestFindLeaksRetry(t *testing.T) {
+	// for i := 0; i < 10; i++ {
+	bg := startBlockedG()
+	require.Error(t, FindLeaks(testOptions()), "Should find leaks with leaked goroutine")
+
+	go func() {
+		time.Sleep(time.Millisecond)
+		bg.unblock()
+	}()
+	require.NoError(t, FindLeaks(), "FindLeaks should retry while background goroutine ends")
+}
+
+type fakeT struct {
+	errors []string
+}
+
+func (ft *fakeT) Error(args ...interface{}) {
+	ft.errors = append(ft.errors, fmt.Sprint(args))
+}
+
+func TestVerifyNoLeaks(t *testing.T) {
+	ft := &fakeT{}
+	VerifyNoLeaks(ft)
+	require.Empty(t, ft.errors, "Expect no errors from VerifyNoLeaks")
+
+	bg := startBlockedG()
+	VerifyNoLeaks(ft, testOptions())
+	require.NotEmpty(t, ft.errors, "Expect errors from VerifyNoLeaks on leaked goroutine")
+	bg.unblock()
+}

gateway/vendor/go.uber.org/goleak/options.go

@@ -0,0 +1,141 @@
+// Copyright (c) 2017 Uber Technologies, Inc.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+package goleak
+
+import (
+	"strings"
+	"time"
+
+	"go.uber.org/goleak/internal/stack"
+)
+
+// Option lets users specify custom verifications.
+type Option interface {
+	apply(*opts)
+}
+
+// We retry up to 20 times if we can't find the goroutine that
+// we are looking for. In between each attempt, we will sleep for
+// a short while to let any running goroutines complete.
+const _defaultRetries = 20
+
+type opts struct {
+	filters    []func(stack.Stack) bool
+	maxRetries int
+	maxSleep   time.Duration
+}
+
+// optionFunc lets us easily write options without a custom type.
+type optionFunc func(*opts)
+
+func (f optionFunc) apply(opts *opts) { f(opts) }
+
+// IgnoreTopFunction ignores any goroutines where the specified function
+// is at the top of the stack. The function name should be fully qualified,
+// e.g., go.uber.org/goleak.IgnoreTopFunction
+func IgnoreTopFunction(f string) Option {
+	return addFilter(func(s stack.Stack) bool {
+		return s.FirstFunction() == f
+	})
+}
+
+func maxSleep(d time.Duration) Option {
+	return optionFunc(func(opts *opts) {
+		opts.maxSleep = d
+	})
+}
+
+func addFilter(f func(stack.Stack) bool) Option {
+	return optionFunc(func(opts *opts) {
+		opts.filters = append(opts.filters, f)
+	})
+}
+
+func buildOpts(options ...Option) *opts {
+	opts := &opts{
+		maxRetries: _defaultRetries,
+		maxSleep:   100 * time.Millisecond,
+	}
+	opts.filters = append(opts.filters,
+		isTestStack,
+		isSyscallStack,
+		isStdLibStack,
+	)
+	for _, option := range options {
+		option.apply(opts)
+	}
+	return opts
+}
+
+func (vo *opts) filter(s stack.Stack) bool {
+	for _, filter := range vo.filters {
+		if filter(s) {
+			return true
+		}
+	}
+	return false
+}
+
+func (vo *opts) retry(i int) bool {
+	if i >= vo.maxRetries {
+		return false
+	}
+
+	d := time.Duration(int(time.Microsecond) << uint(i))
+	if d > vo.maxSleep {
+		d = vo.maxSleep
+	}
+	time.Sleep(d)
+	return true
+}
+
+// isTestStack is a default filter installed to automatically skip goroutines
+// that the testing package runs while the user's tests are running.
+func isTestStack(s stack.Stack) bool {
+	// Until go1.7, the main goroutine ran RunTests, which started
+	// the test in a separate goroutine and waited for that test goroutine
+	// to end by waiting on a channel.
+	// Since go1.7, a separate goroutine is started to wait for signals.
+	switch s.FirstFunction() {
+	case "testing.RunTests", "testing.(*T).Run":
+		// In pre1.7 and post-1.7, background goroutines started by the testing
+		// package are blocked waiting on a channel.
+		return strings.HasPrefix(s.State(), "chan receive")
+	}
+	return false
+}
+
+func isSyscallStack(s stack.Stack) bool {
+	// Typically runs in the background when code uses CGo:
+	// https://github.com/golang/go/issues/16714
+	return s.FirstFunction() == "runtime.goexit" && strings.HasPrefix(s.State(), "syscall")
+}
+
+func isStdLibStack(s stack.Stack) bool {
+	// Importing os/signal starts a background goroutine.
+	// The name of the function at the top has changed between versions.
+	if f := s.FirstFunction(); f == "os/signal.signal_recv" || f == "os/signal.loop" {
+		return true
+	}
+
+	// Using signal.Notify will start a runtime goroutine.
+	return strings.Contains(s.Full(), "runtime.ensureSigM")
+}

gateway/vendor/go.uber.org/goleak/options_test.go

@@ -0,0 +1,78 @@
+// Copyright (c) 2017 Uber Technologies, Inc.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+package goleak
+
+import (
+	"testing"
+	"time"
+
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+	"go.uber.org/goleak/internal/stack"
+)
+
+func TestOptionsFilters(t *testing.T) {
+	opts := buildOpts()
+	cur := stack.Current()
+	all := stack.All()
+
+	// At least one of these should be the same as current, the others should be filtered out.
+	for _, s := range all {
+		if s.ID() == cur.ID() {
+			require.False(t, opts.filter(s), "Current test running function should not be filtered")
+		} else {
+			require.True(t, opts.filter(s), "Default goroutines should be filtered: %v", s)
+		}
+	}
+
+	defer startBlockedG().unblock()
+
+	// Now the filters should find something that doesn't match a filter.
+	countUnfiltered := func() int {
+		var unmatched int
+		for _, s := range stack.All() {
+			if s.ID() == cur.ID() {
+				continue
+			}
+			if !opts.filter(s) {
+				unmatched++
+			}
+		}
+		return unmatched
+	}
+	require.Equal(t, 1, countUnfiltered(), "Expected blockedG goroutine to not match any filter")
+
+	// If we add an extra filter to ignore blockTill, it shouldn't match.
+	opts = buildOpts(IgnoreTopFunction("go.uber.org/goleak.(*blockedG).run"))
+	require.Zero(t, countUnfiltered(), "blockedG should be filtered out. running: %v", stack.All())
+}
+
+func TestOptionsRetry(t *testing.T) {
+	opts := buildOpts()
+	opts.maxRetries = 50 // initial attempt + 50 retries = 11
+	opts.maxSleep = time.Millisecond
+
+	for i := 0; i < 50; i++ {
+		assert.True(t, opts.retry(i), "Attempt %v/51 should allow retrying", i)
+	}
+	assert.False(t, opts.retry(51), "Attempt 51/51 should not allow retrying")
+	assert.False(t, opts.retry(52), "Attempt 52/51 should not allow retrying")
+}

gateway/vendor/go.uber.org/goleak/signal_test.go

@@ -0,0 +1,45 @@
+// Copyright (c) 2017 Uber Technologies, Inc.
+
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+package goleak_test
+
+// Importing the os/signal package causes a goroutine to be started.
+import (
+	"os"
+	"os/signal"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	"go.uber.org/goleak"
+)
+
+func TestNoLeaks(t *testing.T) {
+	// Just importing the package can cause leaks.
+	require.NoError(t, goleak.FindLeaks(), "Found leaks caused by signal import")
+
+	// Register some signal handlers and ensure there's no leaks.
+	c := make(chan os.Signal)
+	signal.Notify(c, os.Interrupt)
+	require.NoError(t, goleak.FindLeaks(), "Found leaks caused by signal.Notify")
+
+	// Restore all registered signals.
+	signal.Reset(os.Interrupt)
+	require.NoError(t, goleak.FindLeaks(), "Found leaks caused after signal.Reset")
+}

gateway/vendor/go.uber.org/goleak/testmain.go

@@ -0,0 +1,61 @@
+// Copyright (c) 2017 Uber Technologies, Inc.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+package goleak
+
+import (
+	"fmt"
+	"io"
+	"os"
+)
+
+// Variables for stubbing in unit tests.
+var (
+	_osExit             = os.Exit
+	_osStderr io.Writer = os.Stderr
+)
+
+// TestingM is the minimal subset of testing.M that we use.
+type TestingM interface {
+	Run() int
+}
+
+// VerifyTestMain can be used in a TestMain function for package tests to
+// verify that there were no goroutine leaks.
+// To use it, your TestMain function should look like:
+//
+// func TestMain(m *testing.M) {
+//   goleak.VerifyTestMain(m)
+// }
+//
+// This will run all tests as per normal, and if they were successful, look
+// for any goroutine leaks and fail the tests if any leaks were found.
+func VerifyTestMain(m TestingM, options ...Option) {
+	exitCode := m.Run()
+
+	if exitCode == 0 {
+		if err := FindLeaks(options...); err != nil {
+			fmt.Fprintf(_osStderr, "goleak: Errors on successful test run: %v\n", err)
+			exitCode = 1
+		}
+	}
+
+	_osExit(exitCode)
+}

gateway/vendor/go.uber.org/goleak/testmain_test.go

@@ -0,0 +1,78 @@
+// Copyright (c) 2017 Uber Technologies, Inc.
+//
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+package goleak
+
+import (
+	"bytes"
+	"testing"
+
+	"github.com/stretchr/testify/assert"
+)
+
+func init() {
+	clearOSStubs()
+}
+
+func clearOSStubs() {
+	// We don't want to use the real os.Exit or os.Stderr so nil them out.
+	// Tests MUST set them explicitly if they rely on them.
+	_osExit = nil
+	_osStderr = nil
+}
+
+type dummyTestMain int
+
+func (d dummyTestMain) Run() int {
+	return int(d)
+}
+
+func osStubs() (chan int, chan string) {
+	exitCode := make(chan int, 1)
+	stderr := make(chan string, 1)
+
+	buf := &bytes.Buffer{}
+	_osStderr = buf
+	_osExit = func(code int) {
+		exitCode <- code
+		stderr <- buf.String()
+		buf.Reset()
+	}
+	return exitCode, stderr
+}
+
+func TestVerifyTestMain(t *testing.T) {
+	defer clearOSStubs()
+	exitCode, stderr := osStubs()
+
+	blocked := startBlockedG()
+	VerifyTestMain(dummyTestMain(7))
+	assert.Equal(t, 7, <-exitCode, "Exit code should not be modified")
+	assert.NotContains(t, <-stderr, "goleak: Errors", "Ignore leaks on unsuccessful runs")
+
+	VerifyTestMain(dummyTestMain(0))
+	assert.Equal(t, 1, <-exitCode, "Expect error due to leaks on successful runs")
+	assert.Contains(t, <-stderr, "goleak: Errors", "Find leaks on successful runs")
+
+	blocked.unblock()
+	VerifyTestMain(dummyTestMain(0))
+	assert.Equal(t, 0, <-exitCode, "Expect no errors without leaks")
+	assert.NotContains(t, <-stderr, "goleak: Errors", "No errors on successful run without leaks")
+}

gateway/vendor/go.uber.org/goleak/utils_test.go

@@ -0,0 +1,45 @@
+// Copyright (c) 2017 Uber Technologies, Inc.
+
+// Permission is hereby granted, free of charge, to any person obtaining a copy
+// of this software and associated documentation files (the "Software"), to deal
+// in the Software without restriction, including without limitation the rights
+// to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+// copies of the Software, and to permit persons to whom the Software is
+// furnished to do so, subject to the following conditions:
+//
+// The above copyright notice and this permission notice shall be included in
+// all copies or substantial portions of the Software.
+//
+// THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+// IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+// FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+// AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+// LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+// OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+// THE SOFTWARE.
+
+package goleak
+
+type blockedG struct {
+	started chan struct{}
+	wait    chan struct{}
+}
+
+func startBlockedG() *blockedG {
+	bg := &blockedG{
+		started: make(chan struct{}),
+		wait:    make(chan struct{}),
+	}
+	go bg.run()
+	<-bg.started
+	return bg
+}
+
+func (bg *blockedG) run() {
+	close(bg.started)
+	<-bg.wait
+}
+
+func (bg *blockedG) unblock() {
+	close(bg.wait)
+}