Alter graceful shutdown sequence

- the shutdown sequence meant that the kubelet was still passing
work to the watchdog after the HTTP socket was closed. This change
means that the kubelet has a chance to run its check before we
finally stop accepting new connections. It will require some
basic co-ordination between the kubelet's checking period and the
"write_timeout" value in the container.

Tested with Kubernetes on GKE - before the change some Pods were
giving a connection refused error due to them being not detected
as unhealthy. Now I receive 0% error rate even with 20 qps.

Issue was shown by scaling to 20 replicas, starting a test with
hey and then scaling to 1 replica while tailing the logs from the
gateway. Before I saw some 502, now I see just 200s.

Signed-off-by: Alex Ellis (VMware) <alexellis2@gmail.com>

watchdog/handler.go

@@ -14,13 +14,12 @@ import (
 	"path/filepath"
 	"strings"
 	"sync"
+	"sync/atomic"
 	"time"
 
 	"github.com/openfaas/faas/watchdog/types"
 )
 
-var acceptingConnections bool
-
 // buildFunctionInput for a GET method this is an empty byte array.
 func buildFunctionInput(config *WatchdogConfig, r *http.Request) ([]byte, error) {
 	var res []byte
@@ -270,7 +269,8 @@ func createLockFile() (string, error) {
 	path := filepath.Join(os.TempDir(), ".lock")
 	log.Printf("Writing lock-file to: %s\n", path)
 	writeErr := ioutil.WriteFile(path, []byte{}, 0660)
-	acceptingConnections = true
+
+	atomic.StoreInt32(&acceptingConnections, 1)
 
 	return path, writeErr
 }
@@ -279,13 +279,14 @@ func makeHealthHandler() func(http.ResponseWriter, *http.Request) {
 	return func(w http.ResponseWriter, r *http.Request) {
 		switch r.Method {
 		case http.MethodGet:
-			if acceptingConnections == false || lockFilePresent() == false {
-				w.WriteHeader(http.StatusInternalServerError)
+			if atomic.LoadInt32(&acceptingConnections) == 0 || lockFilePresent() == false {
+				w.WriteHeader(http.StatusServiceUnavailable)
 				return
 			}
 
 			w.WriteHeader(http.StatusOK)
 			w.Write([]byte("OK"))
+
 			break
 		default:
 			w.WriteHeader(http.StatusMethodNotAllowed)