From ab2f8e85f358a8d7d32e03b5643d59bc0c094d13 Mon Sep 17 00:00:00 2001
From: Alex <alexellis2@gmail.com>
Date: Sat, 21 Jan 2017 10:11:33 +0000
Subject: [PATCH 01/35] Introduce gateway_function_invocation_total to track
 individual functions Introduce prometheus_alertmanager into stack - have it
 fire into webhook stash

---
 deploy_stack.sh             |  1 -
 docker-compose.yml          | 20 +++++++++--
 gateway/build.sh            |  2 +-
 gateway/metrics/metrics.go  |  1 +
 gateway/server.go           | 11 ++++++
 prometheus/alert.rules      | 15 ++++++++
 prometheus/alertmanager.yml | 68 +++++++++++++++++++++++++++++++++++++
 prometheus/prometheus.yml   |  8 ++---
 8 files changed, 118 insertions(+), 8 deletions(-)
 create mode 100644 prometheus/alert.rules
 create mode 100644 prometheus/alertmanager.yml

diff --git a/deploy_stack.sh b/deploy_stack.sh
index 11d3b1c7..c55fc271 100755
--- a/deploy_stack.sh
+++ b/deploy_stack.sh
@@ -2,4 +2,3 @@
 
 echo "Deploying stack"
 docker stack rm func ;  docker stack deploy func --compose-file docker-compose.yml 
-
diff --git a/docker-compose.yml b/docker-compose.yml
index a297de6e..a59a4fd9 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -7,22 +7,38 @@ services:
             - 8080:8080
         image: alexellis2/faas-gateway:latest-dev
         networks:
-        - functions
+            - functions
 
     prometheus:
         image: quay.io/prometheus/prometheus:latest
         volumes:
             - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml
-        command: "-config.file=/etc/prometheus/prometheus.yml -storage.local.path=/prometheus -storage.local.memory-chunks=10000"
+            - ./prometheus/alert.rules:/etc/prometheus/alert.rules
+
+        command: "-config.file=/etc/prometheus/prometheus.yml -storage.local.path=/prometheus -storage.local.memory-chunks=10000 --alertmanager.url=http://alertmanager:9093"
         ports:
             - 9090:9090
         depends_on:
             - gateway
+            - alertmanager
         environment:
             no_proxy:   "gateway"
         networks:
             - functions
 
+    alertmanager:
+        image: quay.io/prometheus/alertmanager
+        environment:
+            no_proxy:   "gateway"
+        volumes:
+            - ./prometheus/alertmanager.yml:/alertmanager.yml
+        command:
+            - '-config.file=/alertmanager.yml'
+        networks:
+            - functions
+        ports:
+            - 9093:9093
+
     # Sample functions go here.
     webhookstash:
         image: alexellis2/faas-webhookstash:latest
diff --git a/gateway/build.sh b/gateway/build.sh
index 50a753b1..0e5585b6 100755
--- a/gateway/build.sh
+++ b/gateway/build.sh
@@ -10,4 +10,4 @@ docker rm -f gateway_extract
 
 echo Building alexellis2/faas-gateway:latest
 
-docker build -t alexellis2/faas-gateway:latest .
+docker build -t alexellis2/faas-gateway:latest-dev .
diff --git a/gateway/metrics/metrics.go b/gateway/metrics/metrics.go
index c41d3abf..41ba1e46 100644
--- a/gateway/metrics/metrics.go
+++ b/gateway/metrics/metrics.go
@@ -11,6 +11,7 @@ type MetricOptions struct {
 	GatewayRequestsTotal         prometheus.Counter
 	GatewayServerlessServedTotal prometheus.Counter
 	GatewayFunctions             prometheus.Histogram
+	GatewayFunctionInvocation    *prometheus.CounterVec
 }
 
 // PrometheusHandler Bootstraps prometheus for metrics collection
diff --git a/gateway/server.go b/gateway/server.go
index e5e74d53..3d8321ef 100644
--- a/gateway/server.go
+++ b/gateway/server.go
@@ -72,6 +72,8 @@ func isAlexa(requestBody []byte) AlexaRequestBody {
 }
 
 func invokeService(w http.ResponseWriter, r *http.Request, metrics metrics.MetricOptions, service string, requestBody []byte) {
+	metrics.GatewayFunctionInvocation.WithLabelValues(service).Add(1)
+
 	stamp := strconv.FormatInt(time.Now().Unix(), 10)
 
 	start := time.Now()
@@ -171,15 +173,24 @@ func main() {
 		Name: "gateway_functions",
 		Help: "Gateway functions",
 	})
+	GatewayFunctionInvocation := prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "gateway_function_invocation_total",
+			Help: "Individual function metrics",
+		},
+		[]string{"function_name"},
+	)
 
 	prometheus.Register(GatewayRequestsTotal)
 	prometheus.Register(GatewayServerlessServedTotal)
 	prometheus.Register(GatewayFunctions)
+	prometheus.Register(GatewayFunctionInvocation)
 
 	metricsOptions := metrics.MetricOptions{
 		GatewayRequestsTotal:         GatewayRequestsTotal,
 		GatewayServerlessServedTotal: GatewayServerlessServedTotal,
 		GatewayFunctions:             GatewayFunctions,
+		GatewayFunctionInvocation:    GatewayFunctionInvocation,
 	}
 
 	r := mux.NewRouter()
diff --git a/prometheus/alert.rules b/prometheus/alert.rules
new file mode 100644
index 00000000..1e521200
--- /dev/null
+++ b/prometheus/alert.rules
@@ -0,0 +1,15 @@
+ALERT service_down
+  IF up == 0
+
+ALERT APIHighInvocationRate
+  IF rate ( gateway_function_invocation_total [10s] ) > 5
+  FOR 30s
+  ANNOTATIONS {
+    summary = "High invocation total on {{ $labels.instance }}",
+    description =  "High invocation total on {{ $labels.instance }}",
+  }
+  LABELS {
+    service = "gateway",
+    severity = "major",
+    value = "{{$value}}",
+  }
diff --git a/prometheus/alertmanager.yml b/prometheus/alertmanager.yml
new file mode 100644
index 00000000..26c3e1e3
--- /dev/null
+++ b/prometheus/alertmanager.yml
@@ -0,0 +1,68 @@
+global:
+  # The smarthost and SMTP sender used for mail notifications.
+  smtp_smarthost: 'localhost:25'
+  smtp_from: 'alertmanager@example.org'
+  smtp_auth_username: 'alertmanager'
+  smtp_auth_password: 'password'
+  # The auth token for Hipchat.
+  hipchat_auth_token: '1234556789'
+  # Alternative host for Hipchat.
+  hipchat_url: 'https://hipchat.foobar.org/'
+
+# The directory from which notification templates are read.
+templates: 
+- '/etc/alertmanager/template/*.tmpl'
+
+# The root route on which each incoming alert enters.
+route:
+  # The labels by which incoming alerts are grouped together. For example,
+  # multiple alerts coming in for cluster=A and alertname=LatencyHigh would
+  # be batched into a single group.
+  group_by: ['alertname', 'cluster', 'service']
+
+  # When a new group of alerts is created by an incoming alert, wait at
+  # least 'group_wait' to send the initial notification.
+  # This way ensures that you get multiple alerts for the same group that start
+  # firing shortly after another are batched together on the first 
+  # notification.
+  group_wait: 30s
+
+  # When the first notification was sent, wait 'group_interval' to send a batch
+  # of new alerts that started firing for that group.
+  group_interval: 5m
+
+  # If an alert has successfully been sent, wait 'repeat_interval' to
+  # resend them.
+  repeat_interval: 3h 
+
+  # A default receiver
+  receiver: scale-up
+
+  # All the above attributes are inherited by all child routes and can 
+  # overwritten on each.
+
+  # The child route trees.
+  routes:
+  - match:
+    service: gateway
+    receiver: scale-up
+    severity: major
+
+
+# Inhibition rules allow to mute a set of alerts given that another alert is
+# firing.
+# We use this to mute any warning-level notifications if the same alert is 
+# already critical.
+inhibit_rules:
+- source_match:
+    severity: 'critical'
+  target_match:
+    severity: 'warning'
+  # Apply inhibition if the alertname is the same.
+  equal: ['alertname', 'cluster', 'service']
+
+receivers:
+- name: 'scale-up'
+  webhook_configs:
+    - url: http://gateway:8080/function/func_webhookstash
+      send_resolved: true
diff --git a/prometheus/prometheus.yml b/prometheus/prometheus.yml
index ad723850..d288a55a 100644
--- a/prometheus/prometheus.yml
+++ b/prometheus/prometheus.yml
@@ -7,12 +7,12 @@ global:
   # Attach these labels to any time series or alerts when communicating with
   # external systems (federation, remote storage, Alertmanager).
   external_labels:
-      monitor: 'codelab-monitor'
+      monitor: 'faas-monitor'
 
 # Load rules once and periodically evaluate them according to the global 'evaluation_interval'.
 rule_files:
-  # - "first.rules"
-  # - "second.rules"
+    - 'alert.rules'
+
 
 # A scrape configuration containing exactly one endpoint to scrape:
 # Here it's Prometheus itself.
@@ -29,6 +29,6 @@ scrape_configs:
       - targets: ['localhost:9090']
 
   - job_name: "gateway"
-    scrape_interval: "15s"
+    scrape_interval: 5s
     static_configs:
       - targets: ['gateway:8080']

From 3a0739fad00966cc290f5959eac880b060a8a407 Mon Sep 17 00:00:00 2001
From: Alex <alexellis2@gmail.com>
Date: Sat, 21 Jan 2017 10:59:29 +0000
Subject: [PATCH 02/35] Add route /system/alert for scaling.

---
 docker-compose.yml          |  2 +-
 gateway/server.go           | 42 ++++++++++++++++++++++++-------------
 prometheus/alertmanager.yml |  6 +++---
 3 files changed, 31 insertions(+), 19 deletions(-)

diff --git a/docker-compose.yml b/docker-compose.yml
index a59a4fd9..8c4c6f26 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -5,7 +5,7 @@ services:
             - "/var/run/docker.sock:/var/run/docker.sock"
         ports:
             - 8080:8080
-        image: alexellis2/faas-gateway:latest-dev
+        image: alexellis2/faas-gateway:latest-dev1
         networks:
             - functions
 
diff --git a/gateway/server.go b/gateway/server.go
index 3d8321ef..88695f5c 100644
--- a/gateway/server.go
+++ b/gateway/server.go
@@ -44,13 +44,7 @@ type AlexaRequestBody struct {
 	Request AlexaRequest `json:"request"`
 }
 
-func lookupSwarmService(serviceName string) (bool, error) {
-	var c *client.Client
-	var err error
-	c, err = client.NewEnvClient()
-	if err != nil {
-		log.Fatal("Error with Docker client.")
-	}
+func lookupSwarmService(serviceName string, c *client.Client) (bool, error) {
 	fmt.Printf("Resolving: '%s'\n", serviceName)
 	serviceFilter := filters.NewArgs()
 	serviceFilter.Add("name", serviceName)
@@ -106,8 +100,8 @@ func invokeService(w http.ResponseWriter, r *http.Request, metrics metrics.Metri
 	metrics.GatewayFunctions.Observe(seconds)
 }
 
-func lookupInvoke(w http.ResponseWriter, r *http.Request, metrics metrics.MetricOptions, name string) {
-	exists, err := lookupSwarmService(name)
+func lookupInvoke(w http.ResponseWriter, r *http.Request, metrics metrics.MetricOptions, name string, c *client.Client) {
+	exists, err := lookupSwarmService(name, c)
 	if err != nil || exists == false {
 		if err != nil {
 			log.Fatalln(err)
@@ -121,7 +115,7 @@ func lookupInvoke(w http.ResponseWriter, r *http.Request, metrics metrics.Metric
 	}
 }
 
-func makeProxy(metrics metrics.MetricOptions, wildcard bool) http.HandlerFunc {
+func makeProxy(metrics metrics.MetricOptions, wildcard bool, c *client.Client) http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
 		metrics.GatewayRequestsTotal.Inc()
 
@@ -135,9 +129,9 @@ func makeProxy(metrics metrics.MetricOptions, wildcard bool) http.HandlerFunc {
 				vars := mux.Vars(r)
 				name := vars["name"]
 				fmt.Println("invoke by name")
-				lookupInvoke(w, r, metrics, name)
+				lookupInvoke(w, r, metrics, name, c)
 			} else if len(header) > 0 {
-				lookupInvoke(w, r, metrics, header[0])
+				lookupInvoke(w, r, metrics, header[0], c)
 			} else {
 				requestBody, _ := ioutil.ReadAll(r.Body)
 				alexaService := isAlexa(requestBody)
@@ -160,6 +154,15 @@ func makeProxy(metrics metrics.MetricOptions, wildcard bool) http.HandlerFunc {
 	}
 }
 
+func makeAlertHandler(c *client.Client) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		fmt.Println(c)
+		// Todo: parse alert, validate alert and scale up or down function
+
+		fmt.Println("Alert received.")
+	}
+}
+
 func main() {
 	GatewayRequestsTotal := prometheus.NewCounter(prometheus.CounterOpts{
 		Name: "gateway_requests_total",
@@ -193,10 +196,19 @@ func main() {
 		GatewayFunctionInvocation:    GatewayFunctionInvocation,
 	}
 
-	r := mux.NewRouter()
-	r.HandleFunc("/", makeProxy(metricsOptions, false))
+	var c *client.Client
+	var err error
+	c, err = client.NewEnvClient()
+	if err != nil {
+		log.Fatal("Error with Docker client.")
+	}
 
-	r.HandleFunc("/function/{name:[a-zA-Z_]+}", makeProxy(metricsOptions, true))
+	r := mux.NewRouter()
+	r.HandleFunc("/function/{name:[a-zA-Z_]+}", makeProxy(metricsOptions, true, c))
+
+	r.HandleFunc("/system/alert", makeAlertHandler(c))
+
+	r.HandleFunc("/", makeProxy(metricsOptions, false, c))
 
 	metricsHandler := metrics.PrometheusHandler()
 	r.Handle("/metrics", metricsHandler)
diff --git a/prometheus/alertmanager.yml b/prometheus/alertmanager.yml
index 26c3e1e3..95d2f7a3 100644
--- a/prometheus/alertmanager.yml
+++ b/prometheus/alertmanager.yml
@@ -44,9 +44,9 @@ route:
   # The child route trees.
   routes:
   - match:
-    service: gateway
-    receiver: scale-up
-    severity: major
+      service: gateway
+      receiver: scale-up
+      severity: major
 
 
 # Inhibition rules allow to mute a set of alerts given that another alert is

From 8da30eecaa8dc52b8ae7acb4222958792baa9fee Mon Sep 17 00:00:00 2001
From: Alex Ellis <AlexEllis2@gmail.com>
Date: Sat, 21 Jan 2017 14:17:20 +0000
Subject: [PATCH 03/35] Plate gateway on manager

---
 docker-compose.yml | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/docker-compose.yml b/docker-compose.yml
index 8c4c6f26..df8a89c0 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -8,6 +8,9 @@ services:
         image: alexellis2/faas-gateway:latest-dev1
         networks:
             - functions
+        deploy:
+            placement:
+                constraints: [node.role == manager]
 
     prometheus:
         image: quay.io/prometheus/prometheus:latest

From b9bd7c8101e3c651d7eab05e6d0fe240cc9c03de Mon Sep 17 00:00:00 2001
From: Alex Ellis <AlexEllis2@gmail.com>
Date: Sat, 21 Jan 2017 15:50:19 +0000
Subject: [PATCH 04/35] Extract separate files/packages for better visibility.

---
 docker-compose.yml           |   2 +-
 gateway/Dockerfile.build     |   2 +
 gateway/metrics/metrics.go   |  38 +++++++
 gateway/proxy.go             | 132 ++++++++++++++++++++++++
 gateway/requests/requests.go |  23 +++++
 gateway/server.go            | 190 ++---------------------------------
 prometheus/alert.rules       |  10 +-
 7 files changed, 209 insertions(+), 188 deletions(-)
 create mode 100644 gateway/proxy.go
 create mode 100644 gateway/requests/requests.go

diff --git a/docker-compose.yml b/docker-compose.yml
index df8a89c0..8ad8e1cf 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -5,7 +5,7 @@ services:
             - "/var/run/docker.sock:/var/run/docker.sock"
         ports:
             - 8080:8080
-        image: alexellis2/faas-gateway:latest-dev1
+        image: alexellis2/faas-gateway:latest-dev
         networks:
             - functions
         deploy:
diff --git a/gateway/Dockerfile.build b/gateway/Dockerfile.build
index f7498e98..fc9d3d4f 100644
--- a/gateway/Dockerfile.build
+++ b/gateway/Dockerfile.build
@@ -10,7 +10,9 @@ RUN go get -d github.com/docker/docker/api/types \
 WORKDIR /go/src/github.com/alexellis/faas/gateway
 
 COPY metrics metrics
+COPY requests requests
 COPY server.go .
+COPY proxy.go .
 
 RUN find /go/src/github.com/alexellis/faas/gateway/
 RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o app .
diff --git a/gateway/metrics/metrics.go b/gateway/metrics/metrics.go
index 41ba1e46..e28d8773 100644
--- a/gateway/metrics/metrics.go
+++ b/gateway/metrics/metrics.go
@@ -18,3 +18,41 @@ type MetricOptions struct {
 func PrometheusHandler() http.Handler {
 	return prometheus.Handler()
 }
+
+func BuildMetricsOptions() MetricOptions {
+	GatewayRequestsTotal := prometheus.NewCounter(prometheus.CounterOpts{
+		Name: "gateway_requests_total",
+		Help: "Total amount of HTTP requests to the gateway",
+	})
+	GatewayServerlessServedTotal := prometheus.NewCounter(prometheus.CounterOpts{
+		Name: "gateway_serverless_invocation_total",
+		Help: "Total amount of serverless function invocations",
+	})
+	GatewayFunctions := prometheus.NewHistogram(prometheus.HistogramOpts{
+		Name: "gateway_functions",
+		Help: "Gateway functions",
+	})
+	GatewayFunctionInvocation := prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: "gateway_function_invocation_total",
+			Help: "Individual function metrics",
+		},
+		[]string{"function_name"},
+	)
+
+	metricsOptions := MetricOptions{
+		GatewayRequestsTotal:         GatewayRequestsTotal,
+		GatewayServerlessServedTotal: GatewayServerlessServedTotal,
+		GatewayFunctions:             GatewayFunctions,
+		GatewayFunctionInvocation:    GatewayFunctionInvocation,
+	}
+
+	return metricsOptions
+}
+
+func RegisterMetrics(metricsOptions MetricOptions) {
+	prometheus.Register(metricsOptions.GatewayRequestsTotal)
+	prometheus.Register(metricsOptions.GatewayServerlessServedTotal)
+	prometheus.Register(metricsOptions.GatewayFunctions)
+	prometheus.Register(metricsOptions.GatewayFunctionInvocation)
+}
diff --git a/gateway/proxy.go b/gateway/proxy.go
new file mode 100644
index 00000000..1eab535f
--- /dev/null
+++ b/gateway/proxy.go
@@ -0,0 +1,132 @@
+package main
+
+import (
+	"bytes"
+	"context"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/alexellis/faas/gateway/metrics"
+	"github.com/alexellis/faas/gateway/requests"
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/filters"
+	"github.com/docker/docker/client"
+	"github.com/gorilla/mux"
+)
+
+// MakeProxy creates a proxy for HTTP web requests which can be routed to a function.
+func MakeProxy(metrics metrics.MetricOptions, wildcard bool, c *client.Client) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		metrics.GatewayRequestsTotal.Inc()
+
+		if r.Method == "POST" {
+			log.Println(r.Header)
+			header := r.Header["X-Function"]
+			log.Println(header)
+			fmt.Println(wildcard)
+
+			if wildcard == true {
+				vars := mux.Vars(r)
+				name := vars["name"]
+				fmt.Println("invoke by name")
+				lookupInvoke(w, r, metrics, name, c)
+			} else if len(header) > 0 {
+				lookupInvoke(w, r, metrics, header[0], c)
+			} else {
+				requestBody, _ := ioutil.ReadAll(r.Body)
+				alexaService := isAlexa(requestBody)
+				fmt.Println(alexaService)
+
+				if len(alexaService.Session.SessionId) > 0 &&
+					len(alexaService.Session.Application.ApplicationId) > 0 &&
+					len(alexaService.Request.Intent.Name) > 0 {
+
+					fmt.Println("Alexa SDK request found")
+					fmt.Printf("SessionId=%s, Intent=%s, AppId=%s\n", alexaService.Session.SessionId, alexaService.Request.Intent.Name, alexaService.Session.Application.ApplicationId)
+
+					invokeService(w, r, metrics, alexaService.Request.Intent.Name, requestBody)
+				} else {
+					w.WriteHeader(http.StatusBadRequest)
+					w.Write([]byte("Provide an x-function header or a valid Alexa SDK request."))
+				}
+			}
+		}
+	}
+}
+
+func isAlexa(requestBody []byte) requests.AlexaRequestBody {
+	body := requests.AlexaRequestBody{}
+	buf := bytes.NewBuffer(requestBody)
+	fmt.Println(buf)
+	str := buf.String()
+	parts := strings.Split(str, "sessionId")
+	if len(parts) > 1 {
+		json.Unmarshal(requestBody, &body)
+	}
+	return body
+}
+
+func lookupInvoke(w http.ResponseWriter, r *http.Request, metrics metrics.MetricOptions, name string, c *client.Client) {
+	exists, err := lookupSwarmService(name, c)
+	if err != nil || exists == false {
+		if err != nil {
+			log.Fatalln(err)
+		}
+		w.WriteHeader(http.StatusInternalServerError)
+		w.Write([]byte("Error resolving service."))
+	}
+	if exists == true {
+		requestBody, _ := ioutil.ReadAll(r.Body)
+		invokeService(w, r, metrics, name, requestBody)
+	}
+}
+
+func lookupSwarmService(serviceName string, c *client.Client) (bool, error) {
+	fmt.Printf("Resolving: '%s'\n", serviceName)
+	serviceFilter := filters.NewArgs()
+	serviceFilter.Add("name", serviceName)
+	services, err := c.ServiceList(context.Background(), types.ServiceListOptions{Filters: serviceFilter})
+
+	return len(services) > 0, err
+}
+
+func invokeService(w http.ResponseWriter, r *http.Request, metrics metrics.MetricOptions, service string, requestBody []byte) {
+	metrics.GatewayFunctionInvocation.WithLabelValues(service).Add(1)
+
+	stamp := strconv.FormatInt(time.Now().Unix(), 10)
+
+	start := time.Now()
+	buf := bytes.NewBuffer(requestBody)
+	url := "http://" + service + ":" + strconv.Itoa(8080) + "/"
+	fmt.Printf("[%s] Forwarding request to: %s\n", stamp, url)
+	response, err := http.Post(url, "text/plain", buf)
+	if err != nil {
+		log.Println(err)
+		w.WriteHeader(500)
+		buf := bytes.NewBufferString("Can't reach service: " + service)
+		w.Write(buf.Bytes())
+		return
+	}
+
+	responseBody, readErr := ioutil.ReadAll(response.Body)
+	if readErr != nil {
+		fmt.Println(readErr)
+		w.WriteHeader(500)
+		buf := bytes.NewBufferString("Error reading response from service: " + service)
+		w.Write(buf.Bytes())
+		return
+	}
+
+	w.WriteHeader(http.StatusOK)
+	w.Write(responseBody)
+	seconds := time.Since(start).Seconds()
+	fmt.Printf("[%s] took %f seconds\n", stamp, seconds)
+	metrics.GatewayServerlessServedTotal.Inc()
+	metrics.GatewayFunctions.Observe(seconds)
+}
diff --git a/gateway/requests/requests.go b/gateway/requests/requests.go
new file mode 100644
index 00000000..94bfa8a3
--- /dev/null
+++ b/gateway/requests/requests.go
@@ -0,0 +1,23 @@
+package requests
+
+type AlexaSessionApplication struct {
+	ApplicationId string `json:"applicationId"`
+}
+
+type AlexaSession struct {
+	SessionId   string                  `json:"sessionId"`
+	Application AlexaSessionApplication `json:"application"`
+}
+
+type AlexaIntent struct {
+	Name string `json:"name"`
+}
+
+type AlexaRequest struct {
+	Intent AlexaIntent `json:"intent"`
+}
+
+type AlexaRequestBody struct {
+	Session AlexaSession `json:"session"`
+	Request AlexaRequest `json:"request"`
+}
diff --git a/gateway/server.go b/gateway/server.go
index 88695f5c..071a388b 100644
--- a/gateway/server.go
+++ b/gateway/server.go
@@ -1,159 +1,16 @@
 package main
 
 import (
-	"bytes"
-	"context"
 	"fmt"
 	"log"
 	"net/http"
-	"strconv"
-	"strings"
 	"time"
 
-	"io/ioutil"
-
-	"encoding/json"
-
 	"github.com/alexellis/faas/gateway/metrics"
-	"github.com/docker/docker/api/types"
-	"github.com/docker/docker/api/types/filters"
 	"github.com/docker/docker/client"
 	"github.com/gorilla/mux"
-	"github.com/prometheus/client_golang/prometheus"
 )
 
-type AlexaSessionApplication struct {
-	ApplicationId string `json:"applicationId"`
-}
-
-type AlexaSession struct {
-	SessionId   string                  `json:"sessionId"`
-	Application AlexaSessionApplication `json:"application"`
-}
-
-type AlexaIntent struct {
-	Name string `json:"name"`
-}
-
-type AlexaRequest struct {
-	Intent AlexaIntent `json:"intent"`
-}
-
-type AlexaRequestBody struct {
-	Session AlexaSession `json:"session"`
-	Request AlexaRequest `json:"request"`
-}
-
-func lookupSwarmService(serviceName string, c *client.Client) (bool, error) {
-	fmt.Printf("Resolving: '%s'\n", serviceName)
-	serviceFilter := filters.NewArgs()
-	serviceFilter.Add("name", serviceName)
-	services, err := c.ServiceList(context.Background(), types.ServiceListOptions{Filters: serviceFilter})
-
-	return len(services) > 0, err
-}
-
-func isAlexa(requestBody []byte) AlexaRequestBody {
-	body := AlexaRequestBody{}
-	buf := bytes.NewBuffer(requestBody)
-	fmt.Println(buf)
-	str := buf.String()
-	parts := strings.Split(str, "sessionId")
-	if len(parts) > 1 {
-		json.Unmarshal(requestBody, &body)
-	}
-	return body
-}
-
-func invokeService(w http.ResponseWriter, r *http.Request, metrics metrics.MetricOptions, service string, requestBody []byte) {
-	metrics.GatewayFunctionInvocation.WithLabelValues(service).Add(1)
-
-	stamp := strconv.FormatInt(time.Now().Unix(), 10)
-
-	start := time.Now()
-	buf := bytes.NewBuffer(requestBody)
-	url := "http://" + service + ":" + strconv.Itoa(8080) + "/"
-	fmt.Printf("[%s] Forwarding request to: %s\n", stamp, url)
-	response, err := http.Post(url, "text/plain", buf)
-	if err != nil {
-		log.Println(err)
-		w.WriteHeader(500)
-		buf := bytes.NewBufferString("Can't reach service: " + service)
-		w.Write(buf.Bytes())
-		return
-	}
-
-	responseBody, readErr := ioutil.ReadAll(response.Body)
-	if readErr != nil {
-		fmt.Println(readErr)
-		w.WriteHeader(500)
-		buf := bytes.NewBufferString("Error reading response from service: " + service)
-		w.Write(buf.Bytes())
-		return
-	}
-
-	w.WriteHeader(http.StatusOK)
-	w.Write(responseBody)
-	seconds := time.Since(start).Seconds()
-	fmt.Printf("[%s] took %f seconds\n", stamp, seconds)
-	metrics.GatewayServerlessServedTotal.Inc()
-	metrics.GatewayFunctions.Observe(seconds)
-}
-
-func lookupInvoke(w http.ResponseWriter, r *http.Request, metrics metrics.MetricOptions, name string, c *client.Client) {
-	exists, err := lookupSwarmService(name, c)
-	if err != nil || exists == false {
-		if err != nil {
-			log.Fatalln(err)
-		}
-		w.WriteHeader(http.StatusInternalServerError)
-		w.Write([]byte("Error resolving service."))
-	}
-	if exists == true {
-		requestBody, _ := ioutil.ReadAll(r.Body)
-		invokeService(w, r, metrics, name, requestBody)
-	}
-}
-
-func makeProxy(metrics metrics.MetricOptions, wildcard bool, c *client.Client) http.HandlerFunc {
-	return func(w http.ResponseWriter, r *http.Request) {
-		metrics.GatewayRequestsTotal.Inc()
-
-		if r.Method == "POST" {
-			log.Println(r.Header)
-			header := r.Header["X-Function"]
-			log.Println(header)
-			fmt.Println(wildcard)
-
-			if wildcard == true {
-				vars := mux.Vars(r)
-				name := vars["name"]
-				fmt.Println("invoke by name")
-				lookupInvoke(w, r, metrics, name, c)
-			} else if len(header) > 0 {
-				lookupInvoke(w, r, metrics, header[0], c)
-			} else {
-				requestBody, _ := ioutil.ReadAll(r.Body)
-				alexaService := isAlexa(requestBody)
-				fmt.Println(alexaService)
-
-				if len(alexaService.Session.SessionId) > 0 &&
-					len(alexaService.Session.Application.ApplicationId) > 0 &&
-					len(alexaService.Request.Intent.Name) > 0 {
-
-					fmt.Println("Alexa SDK request found")
-					fmt.Printf("SessionId=%s, Intent=%s, AppId=%s\n", alexaService.Session.SessionId, alexaService.Request.Intent.Name, alexaService.Session.Application.ApplicationId)
-
-					invokeService(w, r, metrics, alexaService.Request.Intent.Name, requestBody)
-				} else {
-					w.WriteHeader(http.StatusBadRequest)
-					w.Write([]byte("Provide an x-function header or a valid Alexa SDK request."))
-				}
-			}
-		}
-	}
-}
-
 func makeAlertHandler(c *client.Client) http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
 		fmt.Println(c)
@@ -164,51 +21,20 @@ func makeAlertHandler(c *client.Client) http.HandlerFunc {
 }
 
 func main() {
-	GatewayRequestsTotal := prometheus.NewCounter(prometheus.CounterOpts{
-		Name: "gateway_requests_total",
-		Help: "Total amount of HTTP requests to the gateway",
-	})
-	GatewayServerlessServedTotal := prometheus.NewCounter(prometheus.CounterOpts{
-		Name: "gateway_serverless_invocation_total",
-		Help: "Total amount of serverless function invocations",
-	})
-	GatewayFunctions := prometheus.NewHistogram(prometheus.HistogramOpts{
-		Name: "gateway_functions",
-		Help: "Gateway functions",
-	})
-	GatewayFunctionInvocation := prometheus.NewCounterVec(
-		prometheus.CounterOpts{
-			Name: "gateway_function_invocation_total",
-			Help: "Individual function metrics",
-		},
-		[]string{"function_name"},
-	)
-
-	prometheus.Register(GatewayRequestsTotal)
-	prometheus.Register(GatewayServerlessServedTotal)
-	prometheus.Register(GatewayFunctions)
-	prometheus.Register(GatewayFunctionInvocation)
-
-	metricsOptions := metrics.MetricOptions{
-		GatewayRequestsTotal:         GatewayRequestsTotal,
-		GatewayServerlessServedTotal: GatewayServerlessServedTotal,
-		GatewayFunctions:             GatewayFunctions,
-		GatewayFunctionInvocation:    GatewayFunctionInvocation,
-	}
-
-	var c *client.Client
+	var dockerClient *client.Client
 	var err error
-	c, err = client.NewEnvClient()
+	dockerClient, err = client.NewEnvClient()
 	if err != nil {
 		log.Fatal("Error with Docker client.")
 	}
 
+	metricsOptions := metrics.BuildMetricsOptions()
+	metrics.RegisterMetrics(metricsOptions)
+
 	r := mux.NewRouter()
-	r.HandleFunc("/function/{name:[a-zA-Z_]+}", makeProxy(metricsOptions, true, c))
-
-	r.HandleFunc("/system/alert", makeAlertHandler(c))
-
-	r.HandleFunc("/", makeProxy(metricsOptions, false, c))
+	r.HandleFunc("/function/{name:[a-zA-Z_]+}", MakeProxy(metricsOptions, true, dockerClient))
+	r.HandleFunc("/system/alert", makeAlertHandler(dockerClient))
+	r.HandleFunc("/", MakeProxy(metricsOptions, false, dockerClient))
 
 	metricsHandler := metrics.PrometheusHandler()
 	r.Handle("/metrics", metricsHandler)
diff --git a/prometheus/alert.rules b/prometheus/alert.rules
index 1e521200..68e7ce42 100644
--- a/prometheus/alert.rules
+++ b/prometheus/alert.rules
@@ -4,12 +4,12 @@ ALERT service_down
 ALERT APIHighInvocationRate
   IF rate ( gateway_function_invocation_total [10s] ) > 5
   FOR 30s
-  ANNOTATIONS {
-    summary = "High invocation total on {{ $labels.instance }}",
-    description =  "High invocation total on {{ $labels.instance }}",
-  }
   LABELS {
     service = "gateway",
     severity = "major",
-    value = "{{$value}}",
+    value = "{{$value}}"
   }
+  ANNOTATIONS {
+    summary = "High invocation total on {{ $labels.instance }}",
+    description =  "High invocation total on {{ $labels.instance }}"
+  } 

From ff429ce4931d1600ab5d5d689606a375cf83e4a9 Mon Sep 17 00:00:00 2001
From: Alex Ellis <AlexEllis2@gmail.com>
Date: Sun, 22 Jan 2017 10:45:42 +0000
Subject: [PATCH 05/35] Add static front end + functions endpoint.

---
 docker-compose.yml                 |   3 +-
 gateway/Dockerfile                 |   3 +
 gateway/Dockerfile.build           |   2 +-
 gateway/assets/index.html          | 102 +++++++++++++++++++++++++++++
 gateway/assets/script/bootstrap.js |  21 ++++++
 gateway/assets/style/bootstrap.css |  50 ++++++++++++++
 gateway/build.sh                   |   2 +-
 gateway/server.go                  |  68 +++++++++++++++++--
 prometheus/alertmanager.yml        |   2 +-
 prometheus/test_alert.json         |  47 +++++++++++++
 10 files changed, 291 insertions(+), 9 deletions(-)
 create mode 100644 gateway/assets/index.html
 create mode 100644 gateway/assets/script/bootstrap.js
 create mode 100644 gateway/assets/style/bootstrap.css
 create mode 100644 prometheus/test_alert.json

diff --git a/docker-compose.yml b/docker-compose.yml
index 8ad8e1cf..d8255824 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -5,7 +5,7 @@ services:
             - "/var/run/docker.sock:/var/run/docker.sock"
         ports:
             - 8080:8080
-        image: alexellis2/faas-gateway:latest-dev
+        image: alexellis2/faas-gateway:latest-dev2
         networks:
             - functions
         deploy:
@@ -94,6 +94,7 @@ services:
             fprocess:	"wc"
             no_proxy:   "gateway"
             https_proxy: $https_proxy
+
 networks:
     functions:
         driver: overlay
diff --git a/gateway/Dockerfile b/gateway/Dockerfile
index 6084f1ba..ad71154d 100644
--- a/gateway/Dockerfile
+++ b/gateway/Dockerfile
@@ -1,6 +1,9 @@
 FROM alpine:latest
 
+WORKDIR /root/
+
 COPY gateway    .
+COPY assets assets
 
 EXPOSE 8080
 ENV http_proxy      ""
diff --git a/gateway/Dockerfile.build b/gateway/Dockerfile.build
index fc9d3d4f..1f9c1102 100644
--- a/gateway/Dockerfile.build
+++ b/gateway/Dockerfile.build
@@ -6,6 +6,7 @@ RUN go get -d github.com/docker/docker/api/types \
    && go get -d github.com/docker/docker/client \
    && go get github.com/gorilla/mux \
    && go get github.com/prometheus/client_golang/prometheus
+# RUN go get -d github.com/prometheus/client_model/go
 
 WORKDIR /go/src/github.com/alexellis/faas/gateway
 
@@ -14,5 +15,4 @@ COPY requests requests
 COPY server.go .
 COPY proxy.go .
 
-RUN find /go/src/github.com/alexellis/faas/gateway/
 RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o app .
diff --git a/gateway/assets/index.html b/gateway/assets/index.html
new file mode 100644
index 00000000..d3cd755f
--- /dev/null
+++ b/gateway/assets/index.html
@@ -0,0 +1,102 @@
+<html ng-app="faasGateway" >
+<head>
+          <link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Roboto:300,400,500,700,400italic" />
+        <!--<link rel="stylesheet" href="https://ajax.googleapis.com/ajax/libs/angular_material/1.1.0/angular-material.min.css">-->
+
+    <!-- Angular Material CSS now available via Google CDN; version 1.0.7 used here -->
+    <link rel="stylesheet" href="style/angular-material.min.css">
+
+    <link rel="stylesheet" href="style/bootstrap.css">
+
+  </head>
+
+  <body ng-controller="home">
+
+
+<div layout="column" style="height:100%;" ng-cloak>
+
+  <section id="popupContainer" layout="row" flex>
+    <md-sidenav
+        class="md-sidenav-left"
+        md-component-id="left"
+        md-is-locked-open="$mdMedia('gt-sm')"
+        md-whiteframe="4" layout="column">
+
+      <md-toolbar class="md-theme-indigo">
+        <h1 class="md-toolbar-tools">FaaS Gateway</h1>
+      </md-toolbar>
+      
+      <md-content layout-padding>
+        <md-button ng-click="newFunction()" ng-disabled="isFunctionBeingCreated" class="md-primary">New function</md-button>
+
+        <md-list>
+          <md-list-item ng-switch class="md-3-line" ng-click="showFunction(function)" ng-repeat="function in functions" ng-class="function.name == selectedFunction.name ? 'selected' : false">
+              <md-icon ng-switch-when="true" style="color: blue" md-svg-icon="person"></md-icon>
+              <md-icon ng-switch-when="false" md-svg-icon="person-outline"></md-icon>
+              <p>{{function.name}}</p>
+              <md-divider ng-if="!$last"></md-divider>
+          </md-list-item>
+        </md-list>
+      </md-content>
+    </md-sidenav>
+
+    <md-content flex layout-padding ng-if="!selectedFunction" ng-show="functions.length">
+      <div layout="column" layout-align="top center">
+        <p>Select a function.</p>
+      </div>
+      <div flex></div>
+    </md-content>
+
+    <md-content flex layout-padding ng-if="!functions.length">
+      <div layout="column" layout-align="top center">
+        <p>No functions found in FaaS.</p>
+      </div>
+      <div flex></div>
+    </md-content>
+    <md-content flex layout="column" ng-repeat="function in functions" ng-show="function.name == selectedFunction.name">
+
+        <md-card md-theme="default" md-theme-watch>
+            <md-card-title>
+
+                <md-card-title-text>
+
+                    <span class="md-headline">
+                      {{function.name}}
+                    </span>
+                    <div layout-gt-sm="row">
+                      <md-input-container class="md-icon-float md-block">
+                        <label>Replicas</label>
+                        <input ng-model="function.replicas" type="text" readonly="readonly">
+                      </md-input-container>
+                      <md-input-container class="md-block" flex-gt-sm>
+                        <label>Invocation count</label>
+                        <input ng-model="function.invocationCount" type="text" readonly="readonly">
+                      </md-input-container>
+                    </div>
+
+                    <div layout-gt-sm="row">
+                      <md-input-container class="md-block" flex-gt-sm>
+                        <label>Image</label>
+                        <input ng-model="function.image" type="text" readonly="readonly">
+                      </md-input-container>
+                    </div>
+
+                <md-card-title-text>
+
+            </md-card-title>
+        </md-card>
+
+    </md-content>
+  </section>
+  </div>
+
+
+    <script src="https://ajax.googleapis.com/ajax/libs/angularjs/1.5.5/angular.min.js"></script>
+    <script src="https://ajax.googleapis.com/ajax/libs/angularjs/1.5.5/angular-animate.min.js"></script>
+    <script src="https://ajax.googleapis.com/ajax/libs/angularjs/1.5.5/angular-aria.min.js"></script>
+    <script src="https://ajax.googleapis.com/ajax/libs/angularjs/1.5.5/angular-messages.min.js"></script>
+    <script src="https://ajax.googleapis.com/ajax/libs/angular_material/1.1.0/angular-material.min.js"></script>
+    <script src="script/bootstrap.js"></script>
+  </body>
+
+</html>
diff --git a/gateway/assets/script/bootstrap.js b/gateway/assets/script/bootstrap.js
new file mode 100644
index 00000000..05b2285b
--- /dev/null
+++ b/gateway/assets/script/bootstrap.js
@@ -0,0 +1,21 @@
+"use strict"
+var app = angular.module('faasGateway', ['ngMaterial']);
+app.controller("home", ['$scope', '$log', '$http', '$location', '$timeout', function($scope, $log, $http, $location, $timeout) {
+    $scope.functions = [];
+    $http.get("/system/functions").then(function(response) {
+      $scope.functions = response.data;
+    });
+
+    $scope.showFunction = function(fn) {
+        $scope.selectedFunction = fn;
+    };
+
+    // TODO: popup + form to create new Docker service.
+    $scope.newFunction = function() {
+        $scope.functions.push({
+            name: "f" +($scope.functions.length+2),
+            replicas: 0,
+            invokedCount: 0
+        });
+    };
+}]);
diff --git a/gateway/assets/style/bootstrap.css b/gateway/assets/style/bootstrap.css
new file mode 100644
index 00000000..9180312c
--- /dev/null
+++ b/gateway/assets/style/bootstrap.css
@@ -0,0 +1,50 @@
+@import url('https://fonts.googleapis.com/css?family=Rationale');
+
+
+/*Taken from PWD, remove styles not used.*/
+
+.selected button {
+	background-color: rgba(158,158,158,0.2);
+}
+
+md-card-content.terminal-container {
+    background-color: #000;
+    padding: 0;
+}
+
+.clock {
+    font-family: 'Rationale', sans-serif;
+    font-size: 3.0em;
+    color: #1da4eb;
+    text-align: center;
+}
+
+.welcome {
+    background-color: #e7e7e7;
+}
+
+.welcome > div {
+    text-align: center;
+}
+
+.welcome > div > img {
+    max-width: 100%;
+}
+
+.g-recaptcha div {
+    margin-left: auto;
+    margin-right: auto;
+    margin-bottom: auto;
+    margin-top: 50px;
+}
+
+.disconnected {
+    background-color: #FDF4B6;
+}
+md-input-container {
+    margin-bottom: 0;
+}
+md-input-container .md-errors-spacer {
+    height: 0;
+    min-height: 0;
+}
diff --git a/gateway/build.sh b/gateway/build.sh
index 0e5585b6..ee340c32 100755
--- a/gateway/build.sh
+++ b/gateway/build.sh
@@ -10,4 +10,4 @@ docker rm -f gateway_extract
 
 echo Building alexellis2/faas-gateway:latest
 
-docker build -t alexellis2/faas-gateway:latest-dev .
+docker build -t alexellis2/faas-gateway:latest-dev2 .
diff --git a/gateway/server.go b/gateway/server.go
index 071a388b..44e96169 100644
--- a/gateway/server.go
+++ b/gateway/server.go
@@ -1,22 +1,78 @@
 package main
 
 import (
+	"context"
+	"encoding/json"
 	"fmt"
+	"io/ioutil"
 	"log"
 	"net/http"
 	"time"
 
 	"github.com/alexellis/faas/gateway/metrics"
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/filters"
 	"github.com/docker/docker/client"
 	"github.com/gorilla/mux"
+	io_prometheus_client "github.com/prometheus/client_model/go"
 )
 
-func makeAlertHandler(c *client.Client) http.HandlerFunc {
+func makeAlertHandler() http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
-		fmt.Println(c)
+		fmt.Println("Alert received.")
+		body, _ := ioutil.ReadAll(r.Body)
+		fmt.Println(string(body))
 		// Todo: parse alert, validate alert and scale up or down function
 
-		fmt.Println("Alert received.")
+		w.WriteHeader(http.StatusOK)
+	}
+}
+
+// Function exported for system/functions endpoint
+type Function struct {
+	Name            string  `json:"name"`
+	Image           string  `json:"image"`
+	InvocationCount float64 `json:"invocationCount"`
+	Replicas        uint64  `json:"replicas"`
+}
+
+func makeFunctionReader(metricsOptions metrics.MetricOptions, c *client.Client) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+
+		serviceFilter := filters.NewArgs()
+
+		options := types.ServiceListOptions{
+			Filters: serviceFilter,
+		}
+
+		services, err := c.ServiceList(context.Background(), options)
+		if err != nil {
+			fmt.Println(err)
+		}
+
+		// TODO: Filter only "faas" functions (via metadata?)
+
+		functions := make([]Function, 0)
+		for _, service := range services {
+			counter, _ := metricsOptions.GatewayFunctionInvocation.GetMetricWithLabelValues(service.Spec.Name)
+
+			var pbmetric io_prometheus_client.Metric
+			counter.Write(&pbmetric)
+			invocations := pbmetric.GetCounter().GetValue()
+
+			f := Function{
+				Name:            service.Spec.Name,
+				Image:           service.Spec.TaskTemplate.ContainerSpec.Image,
+				InvocationCount: invocations,
+				Replicas:        *service.Spec.Mode.Replicated.Replicas,
+			}
+			functions = append(functions, f)
+		}
+
+		functionBytes, _ := json.Marshal(functions)
+		w.Header().Set("Content-Type", "application/json")
+		w.WriteHeader(200)
+		w.Write(functionBytes)
 	}
 }
 
@@ -33,12 +89,14 @@ func main() {
 
 	r := mux.NewRouter()
 	r.HandleFunc("/function/{name:[a-zA-Z_]+}", MakeProxy(metricsOptions, true, dockerClient))
-	r.HandleFunc("/system/alert", makeAlertHandler(dockerClient))
-	r.HandleFunc("/", MakeProxy(metricsOptions, false, dockerClient))
+	r.HandleFunc("/system/alert", makeAlertHandler())
+	r.HandleFunc("/system/functions", makeFunctionReader(metricsOptions, dockerClient)).Methods("GET")
+	r.HandleFunc("/", MakeProxy(metricsOptions, false, dockerClient)).Methods("POST")
 
 	metricsHandler := metrics.PrometheusHandler()
 	r.Handle("/metrics", metricsHandler)
 
+	r.PathPrefix("/").Handler(http.FileServer(http.Dir("./assets/"))).Methods("GET")
 	s := &http.Server{
 		Addr:           ":8080",
 		ReadTimeout:    8 * time.Second,
diff --git a/prometheus/alertmanager.yml b/prometheus/alertmanager.yml
index 95d2f7a3..8f647258 100644
--- a/prometheus/alertmanager.yml
+++ b/prometheus/alertmanager.yml
@@ -64,5 +64,5 @@ inhibit_rules:
 receivers:
 - name: 'scale-up'
   webhook_configs:
-    - url: http://gateway:8080/function/func_webhookstash
+    - url: http://gateway:8080/system/alert
       send_resolved: true
diff --git a/prometheus/test_alert.json b/prometheus/test_alert.json
new file mode 100644
index 00000000..13f5aa52
--- /dev/null
+++ b/prometheus/test_alert.json
@@ -0,0 +1,47 @@
+{
+   "receiver":"scale-up",
+   "status":"firing",
+   "alerts":[
+      {
+         "status":"firing",
+         "labels":{
+            "alertname":"APIHighInvocationRate",
+            "function_name":"func_echoit",
+            "instance":"gateway:8080",
+            "job":"gateway",
+            "monitor":"faas-monitor",
+            "service":"gateway",
+            "severity":"major",
+            "value":"8"
+         },
+         "annotations":{
+            "description":"High invocation total on gateway:8080",
+            "summary":"High invocation total on gateway:8080"
+         },
+         "startsAt":"2017-01-22T10:40:52.804Z",
+         "endsAt":"0001-01-01T00:00:00Z",
+         "generatorURL":"http://bb1b23e87070:9090/graph?g0.expr=rate%28gateway_function_invocation_total%5B10s%5D%29+%3E+5\u0026g0.tab=0"
+      }
+   ],
+   "groupLabels":{
+      "alertname":"APIHighInvocationRate",
+      "service":"gateway"
+   },
+   "commonLabels":{
+      "alertname":"APIHighInvocationRate",
+      "function_name":"func_echoit",
+      "instance":"gateway:8080",
+      "job":"gateway",
+      "monitor":"faas-monitor",
+      "service":"gateway",
+      "severity":"major",
+      "value":"8"
+   },
+   "commonAnnotations":{
+      "description":"High invocation total on gateway:8080",
+      "summary":"High invocation total on gateway:8080"
+   },
+   "externalURL":"http://c052c835bcee:9093",
+   "version":"3",
+   "groupKey":18195285354214864953
+}
\ No newline at end of file

From 158c4122518302854c1cd25beeab056445957789 Mon Sep 17 00:00:00 2001
From: Alex Ellis <AlexEllis2@gmail.com>
Date: Sun, 22 Jan 2017 11:09:43 +0000
Subject: [PATCH 06/35] Initial test for alert unmarshall

---
 gateway/requests/requests.go                  |  5 ++++
 {prometheus => gateway/tests}/test_alert.json |  0
 gateway/tests/unmarshall_test.go              | 29 +++++++++++++++++++
 3 files changed, 34 insertions(+)
 rename {prometheus => gateway/tests}/test_alert.json (100%)
 create mode 100644 gateway/tests/unmarshall_test.go

diff --git a/gateway/requests/requests.go b/gateway/requests/requests.go
index 94bfa8a3..fb40469c 100644
--- a/gateway/requests/requests.go
+++ b/gateway/requests/requests.go
@@ -21,3 +21,8 @@ type AlexaRequestBody struct {
 	Session AlexaSession `json:"session"`
 	Request AlexaRequest `json:"request"`
 }
+
+type PrometheusAlert struct {
+	Status   string `json:"status"`
+	Receiver string `json:"receiver"`
+}
diff --git a/prometheus/test_alert.json b/gateway/tests/test_alert.json
similarity index 100%
rename from prometheus/test_alert.json
rename to gateway/tests/test_alert.json
diff --git a/gateway/tests/unmarshall_test.go b/gateway/tests/unmarshall_test.go
new file mode 100644
index 00000000..989dcd33
--- /dev/null
+++ b/gateway/tests/unmarshall_test.go
@@ -0,0 +1,29 @@
+package requests
+
+import (
+	"fmt"
+	"testing"
+
+	"io/ioutil"
+
+	"encoding/json"
+
+	"github.com/alexellis/faas/gateway/requests"
+)
+
+func TestUnmarshallAlert(t *testing.T) {
+	file, _ := ioutil.ReadFile("./test_alert.json")
+
+	var alert requests.PrometheusAlert
+	err := json.Unmarshal(file, &alert)
+	if err != nil {
+		t.Fatal(err)
+	}
+	fmt.Println("OK", string(file), alert)
+	if (len(alert.Status)) == 0 {
+		t.Fatal("No status read")
+	}
+	if (len(alert.Receiver)) == 0 {
+		t.Fatal("No status read")
+	}
+}

From cf2317696d5d2d64453699788bfea2bca32e5790 Mon Sep 17 00:00:00 2001
From: Alex Ellis <AlexEllis2@gmail.com>
Date: Sun, 22 Jan 2017 11:13:45 +0000
Subject: [PATCH 07/35] Unmarshall the service name

---
 gateway/requests/requests.go     | 17 ++++++++++++++---
 gateway/tests/unmarshall_test.go | 10 ++++++++++
 2 files changed, 24 insertions(+), 3 deletions(-)

diff --git a/gateway/requests/requests.go b/gateway/requests/requests.go
index fb40469c..0cfd2f9f 100644
--- a/gateway/requests/requests.go
+++ b/gateway/requests/requests.go
@@ -22,7 +22,18 @@ type AlexaRequestBody struct {
 	Request AlexaRequest `json:"request"`
 }
 
-type PrometheusAlert struct {
-	Status   string `json:"status"`
-	Receiver string `json:"receiver"`
+type PrometheusInnerAlertLabel struct {
+	AlertName    string `json:"alertname"`
+	FunctionName string `json:"function_name"`
+}
+
+type PrometheusInnerAlert struct {
+	Status string                    `json:"status"`
+	Labels PrometheusInnerAlertLabel `json:"labels"`
+}
+
+type PrometheusAlert struct {
+	Status   string                 `json:"status"`
+	Receiver string                 `json:"receiver"`
+	Alerts   []PrometheusInnerAlert `json:"alerts"`
 }
diff --git a/gateway/tests/unmarshall_test.go b/gateway/tests/unmarshall_test.go
index 989dcd33..4fa52288 100644
--- a/gateway/tests/unmarshall_test.go
+++ b/gateway/tests/unmarshall_test.go
@@ -26,4 +26,14 @@ func TestUnmarshallAlert(t *testing.T) {
 	if (len(alert.Receiver)) == 0 {
 		t.Fatal("No status read")
 	}
+	if (len(alert.Alerts)) == 0 {
+		t.Fatal("No alerts read")
+	}
+	if (len(alert.Alerts[0].Labels.AlertName)) == 0 {
+		t.Fatal("No alerts name")
+	}
+	if (len(alert.Alerts[0].Labels.FunctionName)) == 0 {
+		t.Fatal("No function name read")
+	}
+
 }

From 79bc6a54ea2d0e75e98f09a80b7d54b3ddd8a27a Mon Sep 17 00:00:00 2001
From: Alex Ellis <AlexEllis2@gmail.com>
Date: Sun, 22 Jan 2017 14:11:36 +0000
Subject: [PATCH 08/35] Add auto-refresh to UI

---
 docker-compose.yml                 |  2 +-
 gateway/Dockerfile                 |  6 +++---
 gateway/Dockerfile.build           |  9 +++++----
 gateway/assets/index.html          | 11 ++++-------
 gateway/assets/script/bootstrap.js | 16 +++++++++++++---
 gateway/build.sh                   |  2 +-
 6 files changed, 27 insertions(+), 19 deletions(-)

diff --git a/docker-compose.yml b/docker-compose.yml
index d8255824..9b4b382d 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -5,7 +5,7 @@ services:
             - "/var/run/docker.sock:/var/run/docker.sock"
         ports:
             - 8080:8080
-        image: alexellis2/faas-gateway:latest-dev2
+        image: alexellis2/faas-gateway:latest-dev3
         networks:
             - functions
         deploy:
diff --git a/gateway/Dockerfile b/gateway/Dockerfile
index ad71154d..d397da87 100644
--- a/gateway/Dockerfile
+++ b/gateway/Dockerfile
@@ -2,11 +2,11 @@ FROM alpine:latest
 
 WORKDIR /root/
 
-COPY gateway    .
-COPY assets assets
-
 EXPOSE 8080
 ENV http_proxy      ""
 ENV https_proxy     ""
 
+COPY gateway    .
+COPY assets     assets
+
 CMD ["./gateway"]
diff --git a/gateway/Dockerfile.build b/gateway/Dockerfile.build
index 1f9c1102..f0b78f6e 100644
--- a/gateway/Dockerfile.build
+++ b/gateway/Dockerfile.build
@@ -10,9 +10,10 @@ RUN go get -d github.com/docker/docker/api/types \
 
 WORKDIR /go/src/github.com/alexellis/faas/gateway
 
-COPY metrics metrics
-COPY requests requests
-COPY server.go .
-COPY proxy.go .
+COPY metrics	metrics
+COPY requests	requests
+COPY tests	    tests
+COPY server.go	.
+COPY proxy.go	.
 
 RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o app .
diff --git a/gateway/assets/index.html b/gateway/assets/index.html
index d3cd755f..43b0c7ce 100644
--- a/gateway/assets/index.html
+++ b/gateway/assets/index.html
@@ -1,8 +1,7 @@
 <html ng-app="faasGateway" >
 <head>
-          <link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Roboto:300,400,500,700,400italic" />
-        <!--<link rel="stylesheet" href="https://ajax.googleapis.com/ajax/libs/angular_material/1.1.0/angular-material.min.css">-->
-
+    <link rel="stylesheet" href="https://fonts.googleapis.com/css?family=Roboto:300,400,500,700,400italic" />
+    <link rel="stylesheet" href="https://ajax.googleapis.com/ajax/libs/angular_material/1.1.0/angular-material.min.css">
     <!-- Angular Material CSS now available via Google CDN; version 1.0.7 used here -->
     <link rel="stylesheet" href="style/angular-material.min.css">
 
@@ -11,9 +10,7 @@
   </head>
 
   <body ng-controller="home">
-
-
-<div layout="column" style="height:100%;" ng-cloak>
+  <div layout="column" style="height:100%;" ng-cloak>
 
   <section id="popupContainer" layout="row" flex>
     <md-sidenav
@@ -30,7 +27,7 @@
         <md-button ng-click="newFunction()" ng-disabled="isFunctionBeingCreated" class="md-primary">New function</md-button>
 
         <md-list>
-          <md-list-item ng-switch class="md-3-line" ng-click="showFunction(function)" ng-repeat="function in functions" ng-class="function.name == selectedFunction.name ? 'selected' : false">
+          <md-list-item ng-switch class="md-3-line" ng-click="showFunction(function)" ng-repeat="function in functions | orderBy: '-invocationCount'" ng-class="function.name == selectedFunction.name ? 'selected' : false">
               <md-icon ng-switch-when="true" style="color: blue" md-svg-icon="person"></md-icon>
               <md-icon ng-switch-when="false" md-svg-icon="person-outline"></md-icon>
               <p>{{function.name}}</p>
diff --git a/gateway/assets/script/bootstrap.js b/gateway/assets/script/bootstrap.js
index 05b2285b..43ca49e0 100644
--- a/gateway/assets/script/bootstrap.js
+++ b/gateway/assets/script/bootstrap.js
@@ -1,10 +1,18 @@
 "use strict"
 var app = angular.module('faasGateway', ['ngMaterial']);
+
 app.controller("home", ['$scope', '$log', '$http', '$location', '$timeout', function($scope, $log, $http, $location, $timeout) {
     $scope.functions = [];
-    $http.get("/system/functions").then(function(response) {
-      $scope.functions = response.data;
-    });
+
+    setInterval(function() {
+        fetch();
+    }, 1000);
+
+    var fetch = function() {
+        $http.get("/system/functions").then(function(response) {
+            $scope.functions = response.data;
+        });
+    };
 
     $scope.showFunction = function(fn) {
         $scope.selectedFunction = fn;
@@ -18,4 +26,6 @@ app.controller("home", ['$scope', '$log', '$http', '$location', '$timeout', func
             invokedCount: 0
         });
     };
+
+    fetch();
 }]);
diff --git a/gateway/build.sh b/gateway/build.sh
index ee340c32..3f93f69e 100755
--- a/gateway/build.sh
+++ b/gateway/build.sh
@@ -10,4 +10,4 @@ docker rm -f gateway_extract
 
 echo Building alexellis2/faas-gateway:latest
 
-docker build -t alexellis2/faas-gateway:latest-dev2 .
+docker build -t alexellis2/faas-gateway:latest-dev3 .

From f4fde50ca80636276cdf62115535a637df518fb3 Mon Sep 17 00:00:00 2001
From: Alex Ellis <AlexEllis2@gmail.com>
Date: Sun, 22 Jan 2017 20:00:48 +0000
Subject: [PATCH 09/35] Add +5 / -5 Replica scaling in response to AlertManager

---
 gateway/proxy.go       |  2 +-
 gateway/server.go      | 45 ++++++++++++++++++++++++++++++++++++++----
 prometheus/alert.rules |  2 +-
 3 files changed, 43 insertions(+), 6 deletions(-)

diff --git a/gateway/proxy.go b/gateway/proxy.go
index 1eab535f..a426f843 100644
--- a/gateway/proxy.go
+++ b/gateway/proxy.go
@@ -29,7 +29,7 @@ func MakeProxy(metrics metrics.MetricOptions, wildcard bool, c *client.Client) h
 			log.Println(r.Header)
 			header := r.Header["X-Function"]
 			log.Println(header)
-			fmt.Println(wildcard)
+			// fmt.Println(wildcard)
 
 			if wildcard == true {
 				vars := mux.Vars(r)
diff --git a/gateway/server.go b/gateway/server.go
index 44e96169..0443f96e 100644
--- a/gateway/server.go
+++ b/gateway/server.go
@@ -10,6 +10,7 @@ import (
 	"time"
 
 	"github.com/alexellis/faas/gateway/metrics"
+	"github.com/alexellis/faas/gateway/requests"
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/filters"
 	"github.com/docker/docker/client"
@@ -17,14 +18,50 @@ import (
 	io_prometheus_client "github.com/prometheus/client_model/go"
 )
 
-func makeAlertHandler() http.HandlerFunc {
+func makeAlertHandler(c *client.Client) http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
-		fmt.Println("Alert received.")
+		log.Println("Alert received.")
 		body, _ := ioutil.ReadAll(r.Body)
 		fmt.Println(string(body))
 		// Todo: parse alert, validate alert and scale up or down function
 
-		w.WriteHeader(http.StatusOK)
+		var req requests.PrometheusAlert
+		err := json.Unmarshal(body, &req)
+		if err != nil {
+			log.Println(err)
+		}
+
+		if len(req.Alerts) > 0 {
+			serviceName := req.Alerts[0].Labels.FunctionName
+			service, _, _ := c.ServiceInspectWithRaw(context.Background(), serviceName)
+			var replicas uint64
+
+			if req.Status == "firing" {
+				if *service.Spec.Mode.Replicated.Replicas < 20 {
+					replicas = *service.Spec.Mode.Replicated.Replicas + uint64(5)
+				} else {
+					return
+				}
+			} else {
+				replicas = *service.Spec.Mode.Replicated.Replicas - uint64(5)
+				if replicas <= 0 {
+					replicas = 1
+				}
+			}
+			log.Printf("Scaling %s to %d replicas.\n", serviceName, replicas)
+
+			service.Spec.Mode.Replicated.Replicas = &replicas
+			updateOpts := types.ServiceUpdateOptions{}
+			updateOpts.RegistryAuthFrom = types.RegistryAuthFromSpec
+
+			response, updateErr := c.ServiceUpdate(context.Background(), service.ID, service.Version, service.Spec, updateOpts)
+			if updateErr != nil {
+				w.WriteHeader(http.StatusInternalServerError)
+				log.Println(response)
+			}
+
+			w.WriteHeader(http.StatusOK)
+		}
 	}
 }
 
@@ -89,7 +126,7 @@ func main() {
 
 	r := mux.NewRouter()
 	r.HandleFunc("/function/{name:[a-zA-Z_]+}", MakeProxy(metricsOptions, true, dockerClient))
-	r.HandleFunc("/system/alert", makeAlertHandler())
+	r.HandleFunc("/system/alert", makeAlertHandler(dockerClient))
 	r.HandleFunc("/system/functions", makeFunctionReader(metricsOptions, dockerClient)).Methods("GET")
 	r.HandleFunc("/", MakeProxy(metricsOptions, false, dockerClient)).Methods("POST")
 
diff --git a/prometheus/alert.rules b/prometheus/alert.rules
index 68e7ce42..05785ddc 100644
--- a/prometheus/alert.rules
+++ b/prometheus/alert.rules
@@ -3,7 +3,7 @@ ALERT service_down
 
 ALERT APIHighInvocationRate
   IF rate ( gateway_function_invocation_total [10s] ) > 5
-  FOR 30s
+  FOR 5s
   LABELS {
     service = "gateway",
     severity = "major",

From 5277cf43a6cdafdc83928d4c6c0cfa78fca77707 Mon Sep 17 00:00:00 2001
From: Alex Ellis <alexellis2@gmail.com>
Date: Sun, 22 Jan 2017 19:19:19 +0000
Subject: [PATCH 10/35] Update TestDrive.md

---
 TestDrive.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/TestDrive.md b/TestDrive.md
index 0f792f12..d53a6aaf 100644
--- a/TestDrive.md
+++ b/TestDrive.md
@@ -4,6 +4,8 @@ FaaS is a platform for building serverless functions on Docker Swarm Mode with f
 
 #### This is a Quickstart guide for the [FaaS functions as a Service](https://github.com/alexellis/faas/) project
 
+> A sample stack with a number of sample functions is provided, you can also clone the code to hack on it or package your own functions.
+
 The guide makes use of a free testing/cloud service, but if you want to try it on your own laptop just follow the guide in the README file on Github. There is also a [blog post](http://blog.alexellis.io/functions-as-a-service/) that goes into the background of the project.
 
 * So let's head over to http://play-with-docker.com/ and start a new session.

From 1d1124ecbf4d5bfae89e997757054ccb0a2ba097 Mon Sep 17 00:00:00 2001
From: Alex Ellis <alexellis2@gmail.com>
Date: Sun, 22 Jan 2017 20:10:32 +0000
Subject: [PATCH 11/35] Update TestDrive.md

---
 TestDrive.md | 12 ++++++++++++
 1 file changed, 12 insertions(+)

diff --git a/TestDrive.md b/TestDrive.md
index d53a6aaf..74ad0d7b 100644
--- a/TestDrive.md
+++ b/TestDrive.md
@@ -44,6 +44,16 @@ Head over to the Github repo now for the quick-start to test out the sample func
 
 Once you're up and running checkout the `gateway_functions_count` metrics on your Prometheus endpoint on *port 9090*.
 
+### More resources:
+
+FaaS is still expanding and growing, check out the development branch for:
+
+* Auto-scaling
+* Prometheus alerts
+* More sample functions
+
+[Development branch](https://github.com/alexellis/faas/tree/labels_metrics)
+
 #### Would you prefer a video overview?
 
 See how to deploy FaaS onto play-with-docker.com and Docker Swarm in 1-2 minutes. See the sample functions in action and watch the graphs in Prometheus as we ramp up the amount of requests. 
@@ -56,3 +66,5 @@ Prometheus is built into FaaS and the sample stack, so you can check throughput
 
 ![](https://pbs.twimg.com/media/C2d9IkbXAAI58fz.jpg)
 
+#### Wanna 
+

From 9086e7d86b93a205a73b91b00a8646bcfcb6341c Mon Sep 17 00:00:00 2001
From: Alex Ellis <alexellis2@gmail.com>
Date: Sun, 22 Jan 2017 20:11:38 +0000
Subject: [PATCH 12/35] Update TestDrive.md

---
 TestDrive.md | 3 ++-
 1 file changed, 2 insertions(+), 1 deletion(-)

diff --git a/TestDrive.md b/TestDrive.md
index 74ad0d7b..86a8f9c5 100644
--- a/TestDrive.md
+++ b/TestDrive.md
@@ -4,7 +4,7 @@ FaaS is a platform for building serverless functions on Docker Swarm Mode with f
 
 #### This is a Quickstart guide for the [FaaS functions as a Service](https://github.com/alexellis/faas/) project
 
-> A sample stack with a number of sample functions is provided, you can also clone the code to hack on it or package your own functions.
+> A Docker stack file with a number of sample functions is provided so that you can get up and running within minutes. You can also clone the code to hack on it or package your own functions.
 
 The guide makes use of a free testing/cloud service, but if you want to try it on your own laptop just follow the guide in the README file on Github. There is also a [blog post](http://blog.alexellis.io/functions-as-a-service/) that goes into the background of the project.
 
@@ -51,6 +51,7 @@ FaaS is still expanding and growing, check out the development branch for:
 * Auto-scaling
 * Prometheus alerts
 * More sample functions
+* Brand new UI
 
 [Development branch](https://github.com/alexellis/faas/tree/labels_metrics)
 

From f57acdb249abf693c8427d2e64707a2571e52b59 Mon Sep 17 00:00:00 2001
From: Alex Ellis <alexellis2@gmail.com>
Date: Sun, 22 Jan 2017 20:14:05 +0000
Subject: [PATCH 13/35] Update TestDrive.md

---
 TestDrive.md | 5 ++---
 1 file changed, 2 insertions(+), 3 deletions(-)

diff --git a/TestDrive.md b/TestDrive.md
index 86a8f9c5..93529e95 100644
--- a/TestDrive.md
+++ b/TestDrive.md
@@ -48,10 +48,10 @@ Once you're up and running checkout the `gateway_functions_count` metrics on you
 
 FaaS is still expanding and growing, check out the development branch for:
 
-* Auto-scaling
+* [Auto-scaling](https://twitter.com/alexellisuk/status/823262200236277762)
 * Prometheus alerts
 * More sample functions
-* Brand new UI
+* [Brand new UI](https://twitter.com/alexellisuk/status/823262200236277762)
 
 [Development branch](https://github.com/alexellis/faas/tree/labels_metrics)
 
@@ -67,5 +67,4 @@ Prometheus is built into FaaS and the sample stack, so you can check throughput
 
 ![](https://pbs.twimg.com/media/C2d9IkbXAAI58fz.jpg)
 
-#### Wanna 
 

From b68e8021550de1b00eabc0d756b32a8a681352ea Mon Sep 17 00:00:00 2001
From: Alex Ellis <alexellis2@gmail.com>
Date: Sun, 22 Jan 2017 20:19:36 +0000
Subject: [PATCH 14/35] Update TestDrive.md

---
 TestDrive.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/TestDrive.md b/TestDrive.md
index 93529e95..ad2a3e3b 100644
--- a/TestDrive.md
+++ b/TestDrive.md
@@ -53,7 +53,7 @@ FaaS is still expanding and growing, check out the development branch for:
 * More sample functions
 * [Brand new UI](https://twitter.com/alexellisuk/status/823262200236277762)
 
-[Development branch](https://github.com/alexellis/faas/tree/labels_metrics)
+[Development branch](https://github.com/alexellis/faas/commits/labels_metrics)
 
 #### Would you prefer a video overview?
 

From 1135f2fa456d196ee422fb5e4da3ce11449eb590 Mon Sep 17 00:00:00 2001
From: Alex Ellis <alexellis2@gmail.com>
Date: Sun, 22 Jan 2017 20:41:43 +0000
Subject: [PATCH 15/35] Update TestDrive.md

---
 TestDrive.md | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/TestDrive.md b/TestDrive.md
index ad2a3e3b..d82f7fcc 100644
--- a/TestDrive.md
+++ b/TestDrive.md
@@ -1,4 +1,4 @@
-### faas - Functions As A Service
+## faas - Functions As A Service
 
 FaaS is a platform for building serverless functions on Docker Swarm Mode with first class metrics. Any UNIX process can be packaged as a function in FaaS enabling you to consume a range of web events without repetitive boiler-plate coding.
 

From 9a42500deb95870733f6fff04bc826a465003d6e Mon Sep 17 00:00:00 2001
From: Alex <alexellis2@gmail.com>
Date: Sun, 22 Jan 2017 22:13:35 +0000
Subject: [PATCH 16/35] Switch from local binary to ADD via release page

---
 sample-functions/AlpineFunction/Dockerfile    | 3 ++-
 sample-functions/CaptainsIntent/Dockerfile    | 4 +++-
 sample-functions/ChangeColorIntent/Dockerfile | 4 +++-
 sample-functions/DockerHubStats/Dockerfile    | 5 ++++-
 sample-functions/HostnameIntent/Dockerfile    | 4 +++-
 sample-functions/NodeInfo/Dockerfile          | 4 +++-
 sample-functions/WebhookStash/Dockerfile      | 4 +++-
 sample-functions/catservice/Dockerfile        | 5 +++--
 8 files changed, 24 insertions(+), 9 deletions(-)

diff --git a/sample-functions/AlpineFunction/Dockerfile b/sample-functions/AlpineFunction/Dockerfile
index c1546c36..bbb83675 100644
--- a/sample-functions/AlpineFunction/Dockerfile
+++ b/sample-functions/AlpineFunction/Dockerfile
@@ -1,6 +1,7 @@
 FROM alpine:latest
 
-COPY fwatchdog  /usr/bin/
+ADD https://github.com/alexellis/faas/releases/download/v0.1-alpha/fwatchdog /usr/bin
+RUN chmod +x /usr/bin/fwatchdog
 
 # Populate example here
 # ENV fprocess="wc -l"
diff --git a/sample-functions/CaptainsIntent/Dockerfile b/sample-functions/CaptainsIntent/Dockerfile
index 5e5a08cb..2a96cd1b 100644
--- a/sample-functions/CaptainsIntent/Dockerfile
+++ b/sample-functions/CaptainsIntent/Dockerfile
@@ -1,6 +1,8 @@
 FROM alpine:latest
 RUN apk --update add nodejs
-COPY ./fwatchdog /usr/bin/
+
+ADD https://github.com/alexellis/faas/releases/download/v0.1-alpha/fwatchdog /usr/bin
+RUN chmod +x /usr/bin/fwatchdog
 
 COPY package.json   .
 COPY handler.js     .
diff --git a/sample-functions/ChangeColorIntent/Dockerfile b/sample-functions/ChangeColorIntent/Dockerfile
index cbc833c2..54450d36 100644
--- a/sample-functions/ChangeColorIntent/Dockerfile
+++ b/sample-functions/ChangeColorIntent/Dockerfile
@@ -1,6 +1,8 @@
 FROM alpine:latest
 RUN apk --update add nodejs
-COPY ./fwatchdog /usr/bin/
+
+ADD https://github.com/alexellis/faas/releases/download/v0.1-alpha/fwatchdog /usr/bin
+RUN chmod +x /usr/bin/fwatchdog
 
 COPY package.json   .
 COPY handler.js     .
diff --git a/sample-functions/DockerHubStats/Dockerfile b/sample-functions/DockerHubStats/Dockerfile
index 6ab0b2bc..7586cc92 100644
--- a/sample-functions/DockerHubStats/Dockerfile
+++ b/sample-functions/DockerHubStats/Dockerfile
@@ -7,7 +7,10 @@ WORKDIR /go/src/github.com/alexellis/faas/sample-functions/DockerHubStats
 COPY . /go/src/github.com/alexellis/faas/sample-functions/DockerHubStats
 
 RUN make
-COPY fwatchdog /usr/bin/fwatchdog
+
+ADD https://github.com/alexellis/faas/releases/download/v0.1-alpha/fwatchdog /usr/bin
+RUN chmod +x /usr/bin/fwatchdog
+
 ENV fprocess "/go/bin/DockerHubStats"
 
 CMD [ "/usr/bin/fwatchdog"]
diff --git a/sample-functions/HostnameIntent/Dockerfile b/sample-functions/HostnameIntent/Dockerfile
index 8866b5ff..0c934127 100644
--- a/sample-functions/HostnameIntent/Dockerfile
+++ b/sample-functions/HostnameIntent/Dockerfile
@@ -1,6 +1,8 @@
 FROM alpine:latest
 RUN apk --update add nodejs
-COPY ./fwatchdog /usr/bin/
+
+ADD https://github.com/alexellis/faas/releases/download/v0.1-alpha/fwatchdog /usr/bin
+RUN chmod +x /usr/bin/fwatchdog
 
 COPY package.json   .
 COPY handler.js     .
diff --git a/sample-functions/NodeInfo/Dockerfile b/sample-functions/NodeInfo/Dockerfile
index 03a7fcd6..b44946be 100644
--- a/sample-functions/NodeInfo/Dockerfile
+++ b/sample-functions/NodeInfo/Dockerfile
@@ -1,6 +1,8 @@
 FROM alpine:latest
 RUN apk --update add nodejs
-COPY ./fwatchdog /usr/bin/
+
+ADD https://github.com/alexellis/faas/releases/download/v0.1-alpha/fwatchdog /usr/bin
+RUN chmod +x /usr/bin/fwatchdog
 
 COPY package.json .
 COPY main.js .
diff --git a/sample-functions/WebhookStash/Dockerfile b/sample-functions/WebhookStash/Dockerfile
index fda40f3e..9816f297 100644
--- a/sample-functions/WebhookStash/Dockerfile
+++ b/sample-functions/WebhookStash/Dockerfile
@@ -5,7 +5,9 @@ WORKDIR /go/src/app
 RUN go get -d -v
 RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o app .
 
-COPY fwatchdog  /usr/bin/
+ADD https://github.com/alexellis/faas/releases/download/v0.1-alpha/fwatchdog /usr/bin
+RUN chmod +x /usr/bin/fwatchdog
+# COPY fwatchdog  /usr/bin/
 
 ENV fprocess="/go/src/app/app"
 CMD ["fwatchdog"]
diff --git a/sample-functions/catservice/Dockerfile b/sample-functions/catservice/Dockerfile
index 30ba1367..24b37abd 100644
--- a/sample-functions/catservice/Dockerfile
+++ b/sample-functions/catservice/Dockerfile
@@ -1,9 +1,10 @@
 FROM alpine:latest
 ENTRYPOINT []
 
-COPY ./fwatchdog /usr/bin/fwatchdog
-ENV fprocess "cat"
+ADD https://github.com/alexellis/faas/releases/download/v0.1-alpha/fwatchdog /usr/bin
+RUN chmod +x /usr/bin/fwatchdog
 
+ENV fprocess "/bin/cat"
 
 EXPOSE 8080
 CMD ["/usr/bin/fwatchdog"]

From 2759aaf849b79c39d420042a1c3d7232e05ac5f2 Mon Sep 17 00:00:00 2001
From: Alex <alexellis2@gmail.com>
Date: Sun, 22 Jan 2017 22:18:42 +0000
Subject: [PATCH 17/35] Remove -l switch from wc

---
 sample-functions/WordCountFunction/Dockerfile | 6 ++----
 1 file changed, 2 insertions(+), 4 deletions(-)

diff --git a/sample-functions/WordCountFunction/Dockerfile b/sample-functions/WordCountFunction/Dockerfile
index 0a579714..34f5055f 100644
--- a/sample-functions/WordCountFunction/Dockerfile
+++ b/sample-functions/WordCountFunction/Dockerfile
@@ -1,7 +1,5 @@
-FROM alexellis2/faas-alpinefunction
-
-COPY fwatchdog  /usr/bin/
+FROM alexellis2/faas-alpinefunction:latest
 
 # Populate example here
-ENV fprocess="wc -l"
+ENV fprocess="wc"
 CMD ["fwatchdog"]

From 0dfafe99c5910f1b8ca9e9774f8932380555b541 Mon Sep 17 00:00:00 2001
From: Alex <alexellis2@gmail.com>
Date: Mon, 23 Jan 2017 09:19:52 +0000
Subject: [PATCH 18/35] Error handling around Docker socket, refactor scaling
 to separate method

---
 gateway/requests/requests.go | 10 ++++
 gateway/server.go            | 99 ++++++++++++++++++++----------------
 2 files changed, 66 insertions(+), 43 deletions(-)

diff --git a/gateway/requests/requests.go b/gateway/requests/requests.go
index 0cfd2f9f..d0cf4512 100644
--- a/gateway/requests/requests.go
+++ b/gateway/requests/requests.go
@@ -17,6 +17,7 @@ type AlexaRequest struct {
 	Intent AlexaIntent `json:"intent"`
 }
 
+// AlexaRequestBody top-level request produced by Alexa SDK
 type AlexaRequestBody struct {
 	Session AlexaSession `json:"session"`
 	Request AlexaRequest `json:"request"`
@@ -32,8 +33,17 @@ type PrometheusInnerAlert struct {
 	Labels PrometheusInnerAlertLabel `json:"labels"`
 }
 
+// PrometheusAlert as produced by AlertManager
 type PrometheusAlert struct {
 	Status   string                 `json:"status"`
 	Receiver string                 `json:"receiver"`
 	Alerts   []PrometheusInnerAlert `json:"alerts"`
 }
+
+// Function exported for system/functions endpoint
+type Function struct {
+	Name            string  `json:"name"`
+	Image           string  `json:"image"`
+	InvocationCount float64 `json:"invocationCount"`
+	Replicas        uint64  `json:"replicas"`
+}
diff --git a/gateway/server.go b/gateway/server.go
index 0443f96e..377f53ac 100644
--- a/gateway/server.go
+++ b/gateway/server.go
@@ -18,12 +18,49 @@ import (
 	io_prometheus_client "github.com/prometheus/client_model/go"
 )
 
+func scaleService(req requests.PrometheusAlert, c *client.Client) error {
+	var err error
+	//Todo: convert to loop / handler.
+	serviceName := req.Alerts[0].Labels.FunctionName
+	service, _, inspectErr := c.ServiceInspectWithRaw(context.Background(), serviceName)
+	if inspectErr != nil {
+		var replicas uint64
+
+		if req.Status == "firing" {
+			if *service.Spec.Mode.Replicated.Replicas < 20 {
+				replicas = *service.Spec.Mode.Replicated.Replicas + uint64(5)
+			} else {
+				return err
+			}
+		} else {
+			replicas = *service.Spec.Mode.Replicated.Replicas - uint64(5)
+			if replicas <= 0 {
+				replicas = 1
+			}
+		}
+		log.Printf("Scaling %s to %d replicas.\n", serviceName, replicas)
+
+		service.Spec.Mode.Replicated.Replicas = &replicas
+		updateOpts := types.ServiceUpdateOptions{}
+		updateOpts.RegistryAuthFrom = types.RegistryAuthFromSpec
+
+		response, updateErr := c.ServiceUpdate(context.Background(), service.ID, service.Version, service.Spec, updateOpts)
+		if updateErr != nil {
+			err = updateErr
+		}
+		log.Println(response)
+
+	} else {
+		err = inspectErr
+	}
+
+	return err
+}
+
 func makeAlertHandler(c *client.Client) http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
 		log.Println("Alert received.")
 		body, _ := ioutil.ReadAll(r.Body)
-		fmt.Println(string(body))
-		// Todo: parse alert, validate alert and scale up or down function
 
 		var req requests.PrometheusAlert
 		err := json.Unmarshal(body, &req)
@@ -32,47 +69,18 @@ func makeAlertHandler(c *client.Client) http.HandlerFunc {
 		}
 
 		if len(req.Alerts) > 0 {
-			serviceName := req.Alerts[0].Labels.FunctionName
-			service, _, _ := c.ServiceInspectWithRaw(context.Background(), serviceName)
-			var replicas uint64
-
-			if req.Status == "firing" {
-				if *service.Spec.Mode.Replicated.Replicas < 20 {
-					replicas = *service.Spec.Mode.Replicated.Replicas + uint64(5)
-				} else {
-					return
-				}
-			} else {
-				replicas = *service.Spec.Mode.Replicated.Replicas - uint64(5)
-				if replicas <= 0 {
-					replicas = 1
-				}
-			}
-			log.Printf("Scaling %s to %d replicas.\n", serviceName, replicas)
-
-			service.Spec.Mode.Replicated.Replicas = &replicas
-			updateOpts := types.ServiceUpdateOptions{}
-			updateOpts.RegistryAuthFrom = types.RegistryAuthFromSpec
-
-			response, updateErr := c.ServiceUpdate(context.Background(), service.ID, service.Version, service.Spec, updateOpts)
-			if updateErr != nil {
+			err := scaleService(req, c)
+			if err != nil {
+				log.Println(err)
 				w.WriteHeader(http.StatusInternalServerError)
-				log.Println(response)
+			} else {
+				w.WriteHeader(http.StatusOK)
 			}
-
-			w.WriteHeader(http.StatusOK)
 		}
 	}
 }
 
-// Function exported for system/functions endpoint
-type Function struct {
-	Name            string  `json:"name"`
-	Image           string  `json:"image"`
-	InvocationCount float64 `json:"invocationCount"`
-	Replicas        uint64  `json:"replicas"`
-}
-
+// makeFunctionReader gives a summary of Function structs with Docker service stats overlaid with Prometheus counters.
 func makeFunctionReader(metricsOptions metrics.MetricOptions, c *client.Client) http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
 
@@ -88,16 +96,16 @@ func makeFunctionReader(metricsOptions metrics.MetricOptions, c *client.Client)
 		}
 
 		// TODO: Filter only "faas" functions (via metadata?)
-
-		functions := make([]Function, 0)
+		functions := make([]requests.Function, 0)
 		for _, service := range services {
 			counter, _ := metricsOptions.GatewayFunctionInvocation.GetMetricWithLabelValues(service.Spec.Name)
 
-			var pbmetric io_prometheus_client.Metric
-			counter.Write(&pbmetric)
-			invocations := pbmetric.GetCounter().GetValue()
+			// Get the metric's value from ProtoBuf interface (idea via Julius Volz)
+			var protoMetric io_prometheus_client.Metric
+			counter.Write(&protoMetric)
+			invocations := protoMetric.GetCounter().GetValue()
 
-			f := Function{
+			f := requests.Function{
 				Name:            service.Spec.Name,
 				Image:           service.Spec.TaskTemplate.ContainerSpec.Image,
 				InvocationCount: invocations,
@@ -120,6 +128,11 @@ func main() {
 	if err != nil {
 		log.Fatal("Error with Docker client.")
 	}
+	dockerVersion, err := dockerClient.ServerVersion(context.Background())
+	if err != nil {
+		log.Fatal("Error with Docker server.\n", err)
+	}
+	log.Println("API version: %s, %s\n", dockerVersion.APIVersion, dockerVersion.Version)
 
 	metricsOptions := metrics.BuildMetricsOptions()
 	metrics.RegisterMetrics(metricsOptions)

From 33a8f012f5b762c0b69e0cbb18c29a048c7261c2 Mon Sep 17 00:00:00 2001
From: Alex Ellis <alexellis2@gmail.com>
Date: Mon, 23 Jan 2017 15:26:13 +0000
Subject: [PATCH 19/35] Update README.md

---
 README.md | 12 +++++++++++-
 1 file changed, 11 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index def0aa9b..0614d047 100644
--- a/README.md
+++ b/README.md
@@ -186,7 +186,17 @@ This binary fwatchdog acts as a watchdog for your function. Features:
 * Update quick-start to use `docker stack deploy`
  * Have `docker stack deploy` include a pre-configured Prometheus instance.
 
-## Building a development environment:
+## Development
+
+For development of the FaaS framework / library read on. If you would like to consume the project with your own functions then you can use the public images and the supplied `docker stack` file as a template (docker-compose.yml)
+
+### Contributing
+
+* If you have found a bug please raise an issue.
+* If the documentation can be improved / translated etc please raise an issue to discuss.
+* If you would like to contribute to the codebase please raise an issue to propose the change.
+
+### Building a development environment:
 
 To use multiple hosts you should push your services (functions) to the Docker Hub or a registry accessible to all nodes.
 

From b11c400436c0b2edb8dec0a1b4f7a8d430397d8c Mon Sep 17 00:00:00 2001
From: Alex <alexellis2@gmail.com>
Date: Mon, 23 Jan 2017 18:30:29 +0000
Subject: [PATCH 20/35] Combine stdout/stderr (experimental)

Don't panic on error, keep alive and return 500.
---
 watchdog/main.go | 22 ++++++++++++++++++----
 1 file changed, 18 insertions(+), 4 deletions(-)

diff --git a/watchdog/main.go b/watchdog/main.go
index d22c7a52..dfcde9a2 100644
--- a/watchdog/main.go
+++ b/watchdog/main.go
@@ -1,6 +1,7 @@
 package main
 
 import (
+	"bytes"
 	"io/ioutil"
 	"log"
 	"net/http"
@@ -20,18 +21,31 @@ func main() {
 
 	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
 		if r.Method == "POST" {
+
 			process := os.Getenv("fprocess")
+
 			parts := strings.Split(process, " ")
+
 			targetCmd := exec.Command(parts[0], parts[1:]...)
 			writer, _ := targetCmd.StdinPipe()
+
 			res, _ := ioutil.ReadAll(r.Body)
+
 			writer.Write(res)
 			writer.Close()
-			out, err := targetCmd.Output()
-			if err != nil {
-				panic(err)
-			}
 
+			out, err := targetCmd.Output()
+			targetCmd.CombinedOutput()
+			if err != nil {
+				log.Println(targetCmd, err)
+				w.WriteHeader(500)
+				response := bytes.NewBufferString(err.Error())
+				w.Write(response.Bytes())
+				return
+			}
+			w.WriteHeader(200)
+
+			// TODO: consider stdout to container as configurable via env-variable.
 			os.Stdout.Write(out)
 			w.Write(out)
 		}

From afcf0deb0910a6c2d3ebaa4af76e83230b9c9f77 Mon Sep 17 00:00:00 2001
From: Alex <alexellis2@gmail.com>
Date: Mon, 23 Jan 2017 21:03:08 +0000
Subject: [PATCH 21/35] Publish license

---
 LICENSE   | 22 ++++++++++++++++++++++
 README.md | 10 +++++++++-
 2 files changed, 31 insertions(+), 1 deletion(-)
 create mode 100644 LICENSE

diff --git a/LICENSE b/LICENSE
new file mode 100644
index 00000000..2e041d22
--- /dev/null
+++ b/LICENSE
@@ -0,0 +1,22 @@
+MIT License
+
+Copyright (c) 2010-2017 Alex Ellis
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in
+all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN
+THE SOFTWARE.
+
diff --git a/README.md b/README.md
index 0614d047..5e435fca 100644
--- a/README.md
+++ b/README.md
@@ -190,12 +190,20 @@ This binary fwatchdog acts as a watchdog for your function. Features:
 
 For development of the FaaS framework / library read on. If you would like to consume the project with your own functions then you can use the public images and the supplied `docker stack` file as a template (docker-compose.yml)
 
-### Contributing
+### License
+
+This project is licensed until the MIT License.
+
+## Contributing
+
+Here are a few guidelines for contributing:
 
 * If you have found a bug please raise an issue.
 * If the documentation can be improved / translated etc please raise an issue to discuss.
 * If you would like to contribute to the codebase please raise an issue to propose the change.
 
+> Please provide a summary of what you changed, how you did it and how it can be tested.
+
 ### Building a development environment:
 
 To use multiple hosts you should push your services (functions) to the Docker Hub or a registry accessible to all nodes.

From 6feaff784e4684db8507577be6c5d36daa98e194 Mon Sep 17 00:00:00 2001
From: Alex <alexellis2@gmail.com>
Date: Mon, 23 Jan 2017 21:46:17 +0000
Subject: [PATCH 22/35] Pin gateway services

---
 docker-compose.yml | 10 ++++++++++
 1 file changed, 10 insertions(+)

diff --git a/docker-compose.yml b/docker-compose.yml
index 9b4b382d..0dda03fd 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -1,5 +1,7 @@
 version: "3"
 services:
+
+# Core API services are pinned, HA is provided for functions.
     gateway:
         volumes:
             - "/var/run/docker.sock:/var/run/docker.sock"
@@ -28,6 +30,10 @@ services:
             no_proxy:   "gateway"
         networks:
             - functions
+        deploy:
+            placement:
+                constraints: [node.role == manager]
+
 
     alertmanager:
         image: quay.io/prometheus/alertmanager
@@ -41,6 +47,10 @@ services:
             - functions
         ports:
             - 9093:9093
+        deploy:
+            placement:
+                constraints: [node.role == manager]
+
 
     # Sample functions go here.
     webhookstash:

From 59ca597903654afa577b24362f09ca6800369ee7 Mon Sep 17 00:00:00 2001
From: Alex <alexellis2@gmail.com>
Date: Mon, 23 Jan 2017 22:12:47 +0000
Subject: [PATCH 23/35] Fix if-guard for replica scaling

---
 gateway/server.go | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/gateway/server.go b/gateway/server.go
index 377f53ac..a1b96a16 100644
--- a/gateway/server.go
+++ b/gateway/server.go
@@ -23,7 +23,7 @@ func scaleService(req requests.PrometheusAlert, c *client.Client) error {
 	//Todo: convert to loop / handler.
 	serviceName := req.Alerts[0].Labels.FunctionName
 	service, _, inspectErr := c.ServiceInspectWithRaw(context.Background(), serviceName)
-	if inspectErr != nil {
+	if inspectErr == nil {
 		var replicas uint64
 
 		if req.Status == "firing" {
@@ -60,12 +60,17 @@ func scaleService(req requests.PrometheusAlert, c *client.Client) error {
 func makeAlertHandler(c *client.Client) http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
 		log.Println("Alert received.")
-		body, _ := ioutil.ReadAll(r.Body)
+		body, readErr := ioutil.ReadAll(r.Body)
+		if readErr != nil {
+			log.Println(readErr)
+			return
+		}
 
 		var req requests.PrometheusAlert
 		err := json.Unmarshal(body, &req)
 		if err != nil {
 			log.Println(err)
+			return
 		}
 
 		if len(req.Alerts) > 0 {

From cdd5219200c2325f385153dffcebc63ed9995b51 Mon Sep 17 00:00:00 2001
From: Alex <alexellis2@gmail.com>
Date: Mon, 23 Jan 2017 22:44:03 +0000
Subject: [PATCH 24/35] Tweak alertmanager timeout + simplify down-scale of
 replicas

---
 gateway/server.go           | 15 ++++++++++++---
 prometheus/alertmanager.yml |  7 ++++---
 2 files changed, 16 insertions(+), 6 deletions(-)

diff --git a/gateway/server.go b/gateway/server.go
index a1b96a16..ef39e65d 100644
--- a/gateway/server.go
+++ b/gateway/server.go
@@ -32,12 +32,21 @@ func scaleService(req requests.PrometheusAlert, c *client.Client) error {
 			} else {
 				return err
 			}
-		} else {
-			replicas = *service.Spec.Mode.Replicated.Replicas - uint64(5)
-			if replicas <= 0 {
+		} else { // Resolved event.
+			// Previously decremented by 5, but event only fires once, so set to 1/1.
+			if *service.Spec.Mode.Replicated.Replicas > 1 {
+				// replicas = *service.Spec.Mode.Replicated.Replicas - uint64(5)
+				// if replicas < 1 {
+				// replicas = 1
+				// }
+				// return nil
+
 				replicas = 1
+			} else {
+				return nil
 			}
 		}
+
 		log.Printf("Scaling %s to %d replicas.\n", serviceName, replicas)
 
 		service.Spec.Mode.Replicated.Replicas = &replicas
diff --git a/prometheus/alertmanager.yml b/prometheus/alertmanager.yml
index 8f647258..93282307 100644
--- a/prometheus/alertmanager.yml
+++ b/prometheus/alertmanager.yml
@@ -25,15 +25,15 @@ route:
   # This way ensures that you get multiple alerts for the same group that start
   # firing shortly after another are batched together on the first 
   # notification.
-  group_wait: 30s
+  group_wait: 5s
 
   # When the first notification was sent, wait 'group_interval' to send a batch
   # of new alerts that started firing for that group.
-  group_interval: 5m
+  group_interval: 10s
 
   # If an alert has successfully been sent, wait 'repeat_interval' to
   # resend them.
-  repeat_interval: 3h 
+  repeat_interval: 30s 
 
   # A default receiver
   receiver: scale-up
@@ -66,3 +66,4 @@ receivers:
   webhook_configs:
     - url: http://gateway:8080/system/alert
       send_resolved: true
+

From 533a6230f5548f77189cca99cda9ebbed2fe7ff9 Mon Sep 17 00:00:00 2001
From: Alex <alexellis2@gmail.com>
Date: Mon, 23 Jan 2017 23:26:31 +0000
Subject: [PATCH 25/35] Make timeouts configurable for watchdog process. Make
 debug into container optional.

---
 watchdog/main.go | 55 +++++++++++++++++++++++++++++++++++++++---------
 1 file changed, 45 insertions(+), 10 deletions(-)

diff --git a/watchdog/main.go b/watchdog/main.go
index dfcde9a2..9ecf1fec 100644
--- a/watchdog/main.go
+++ b/watchdog/main.go
@@ -7,23 +7,53 @@ import (
 	"net/http"
 	"os"
 	"os/exec"
+	"strconv"
 	"strings"
 	"time"
 )
 
 func main() {
+	readTimeoutStr := os.Getenv("read_timeout")
+	writeTimeoutStr := os.Getenv("write_timeout")
+	writeDebugStr := os.Getenv("write_debug")
+	process := os.Getenv("fprocess")
+
+	readTimeout := 5 * time.Second
+	writeTimeout := 5 * time.Second
+	writeDebug := true
+
+	if len(process) == 0 {
+		log.Panicln("Provide a valid process via fprocess environmental variable.")
+		return
+	}
+
+	if len(writeDebugStr) > 0 && writeDebugStr == "false" {
+		writeDebug = false
+	}
+
+	if len(readTimeoutStr) > 0 {
+		parsedVal, parseErr := strconv.Atoi(readTimeoutStr)
+		if parseErr == nil && parsedVal > 0 {
+			readTimeout = time.Duration(parsedVal) * time.Second
+		}
+	}
+
+	if len(writeTimeoutStr) > 0 {
+		parsedVal, parseErr := strconv.Atoi(writeTimeoutStr)
+		if parseErr == nil && parsedVal > 0 {
+			writeTimeout = time.Duration(parsedVal) * time.Second
+		}
+	}
+
 	s := &http.Server{
 		Addr:           ":8080",
-		ReadTimeout:    5 * time.Second,
-		WriteTimeout:   5 * time.Second,
+		ReadTimeout:    readTimeout,
+		WriteTimeout:   writeTimeout,
 		MaxHeaderBytes: 1 << 20,
 	}
 
 	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
 		if r.Method == "POST" {
-
-			process := os.Getenv("fprocess")
-
 			parts := strings.Split(process, " ")
 
 			targetCmd := exec.Command(parts[0], parts[1:]...)
@@ -34,10 +64,13 @@ func main() {
 			writer.Write(res)
 			writer.Close()
 
-			out, err := targetCmd.Output()
-			targetCmd.CombinedOutput()
+			out, err := targetCmd.CombinedOutput()
+
 			if err != nil {
-				log.Println(targetCmd, err)
+				if writeDebug == true {
+					log.Println(targetCmd, err)
+				}
+
 				w.WriteHeader(500)
 				response := bytes.NewBufferString(err.Error())
 				w.Write(response.Bytes())
@@ -45,8 +78,10 @@ func main() {
 			}
 			w.WriteHeader(200)
 
-			// TODO: consider stdout to container as configurable via env-variable.
-			os.Stdout.Write(out)
+			if writeDebug == true {
+				os.Stdout.Write(out)
+			}
+
 			w.Write(out)
 		}
 	})

From 5bafacafc93f8813dc29b4dd3f63f274935a87af Mon Sep 17 00:00:00 2001
From: Alex <alexellis2@gmail.com>
Date: Tue, 24 Jan 2017 20:00:15 +0000
Subject: [PATCH 26/35] Add timeouts and unit test config.

---
 watchdog/.gitignore     |  1 +
 watchdog/config_test.go | 91 +++++++++++++++++++++++++++++++++++++++++
 watchdog/main.go        | 81 ++++++++++++++++--------------------
 watchdog/readconfig.go  | 69 +++++++++++++++++++++++++++++++
 4 files changed, 197 insertions(+), 45 deletions(-)
 create mode 100644 watchdog/config_test.go
 create mode 100644 watchdog/readconfig.go

diff --git a/watchdog/.gitignore b/watchdog/.gitignore
index 512d37b0..cd8ffecc 100644
--- a/watchdog/.gitignore
+++ b/watchdog/.gitignore
@@ -1 +1,2 @@
 fwatchdog
+watchdog
diff --git a/watchdog/config_test.go b/watchdog/config_test.go
new file mode 100644
index 00000000..74f2b025
--- /dev/null
+++ b/watchdog/config_test.go
@@ -0,0 +1,91 @@
+package main
+
+import "testing"
+
+type EnvBucket struct {
+	Items map[string]string
+}
+
+func NewEnvBucket() EnvBucket {
+	return EnvBucket{
+		Items: make(map[string]string),
+	}
+}
+
+func (e EnvBucket) Getenv(key string) string {
+	return e.Items[key]
+}
+
+func (e EnvBucket) Setenv(key string, value string) {
+	e.Items[key] = value
+}
+func TestRead_WriteDebug_DefaultIsTrueConfig(t *testing.T) {
+	defaults := NewEnvBucket()
+	readConfig := ReadConfig{}
+
+	config := readConfig.Read(defaults)
+
+	if config.writeDebug != true {
+		t.Logf("writeDebug should have been true")
+		t.Fail()
+	}
+}
+func TestRead_WriteDebug_FalseConfig(t *testing.T) {
+	defaults := NewEnvBucket()
+	readConfig := ReadConfig{}
+	defaults.Setenv("writeDebug", "true")
+
+	config := readConfig.Read(defaults)
+
+	if config.writeDebug != true {
+		t.Logf("writeDebug should have been true")
+		t.Fail()
+	}
+}
+
+func TestRead_FprocessConfig(t *testing.T) {
+	defaults := NewEnvBucket()
+	readConfig := ReadConfig{}
+	defaults.Setenv("fprocess", "cat")
+
+	config := readConfig.Read(defaults)
+
+	if config.faasProcess != "cat" {
+		t.Logf("fprocess envVariable incorrect, got: %s.\n", config.faasProcess)
+		t.Fail()
+	}
+}
+
+func TestRead_EmptyTimeoutConfig(t *testing.T) {
+	defaults := NewEnvBucket()
+	readConfig := ReadConfig{}
+
+	config := readConfig.Read(defaults)
+
+	if (config.readTimeout) != 5 {
+		t.Log("readTimeout incorrect")
+		t.Fail()
+	}
+	if (config.writeTimeout) != 5 {
+		t.Log("writeTimeout incorrect")
+		t.Fail()
+	}
+}
+
+func TestRead_ReadAndWriteTimeoutConfig(t *testing.T) {
+	defaults := NewEnvBucket()
+	defaults.Setenv("read_timeout", "10")
+	defaults.Setenv("write_timeout", "60")
+
+	readConfig := ReadConfig{}
+	config := readConfig.Read(defaults)
+
+	if (config.readTimeout) != 10 {
+		t.Logf("readTimeout incorrect, got: %d\n", config.readTimeout)
+		t.Fail()
+	}
+	if (config.writeTimeout) != 60 {
+		t.Logf("writeTimeout incorrect, got: %d\n", config.writeTimeout)
+		t.Fail()
+	}
+}
diff --git a/watchdog/main.go b/watchdog/main.go
index 9ecf1fec..d608a048 100644
--- a/watchdog/main.go
+++ b/watchdog/main.go
@@ -7,54 +7,23 @@ import (
 	"net/http"
 	"os"
 	"os/exec"
-	"strconv"
 	"strings"
 	"time"
 )
 
-func main() {
-	readTimeoutStr := os.Getenv("read_timeout")
-	writeTimeoutStr := os.Getenv("write_timeout")
-	writeDebugStr := os.Getenv("write_debug")
-	process := os.Getenv("fprocess")
+// OsEnv implements interface to wrap os.Getenv
+type OsEnv struct {
+}
 
-	readTimeout := 5 * time.Second
-	writeTimeout := 5 * time.Second
-	writeDebug := true
+// Getenv wraps os.Getenv
+func (OsEnv) Getenv(key string) string {
+	return os.Getenv(key)
+}
 
-	if len(process) == 0 {
-		log.Panicln("Provide a valid process via fprocess environmental variable.")
-		return
-	}
-
-	if len(writeDebugStr) > 0 && writeDebugStr == "false" {
-		writeDebug = false
-	}
-
-	if len(readTimeoutStr) > 0 {
-		parsedVal, parseErr := strconv.Atoi(readTimeoutStr)
-		if parseErr == nil && parsedVal > 0 {
-			readTimeout = time.Duration(parsedVal) * time.Second
-		}
-	}
-
-	if len(writeTimeoutStr) > 0 {
-		parsedVal, parseErr := strconv.Atoi(writeTimeoutStr)
-		if parseErr == nil && parsedVal > 0 {
-			writeTimeout = time.Duration(parsedVal) * time.Second
-		}
-	}
-
-	s := &http.Server{
-		Addr:           ":8080",
-		ReadTimeout:    readTimeout,
-		WriteTimeout:   writeTimeout,
-		MaxHeaderBytes: 1 << 20,
-	}
-
-	http.HandleFunc("/", func(w http.ResponseWriter, r *http.Request) {
+func makeRequestHandler(config *WatchdogConfig) func(http.ResponseWriter, *http.Request) {
+	return func(w http.ResponseWriter, r *http.Request) {
 		if r.Method == "POST" {
-			parts := strings.Split(process, " ")
+			parts := strings.Split(config.faasProcess, " ")
 
 			targetCmd := exec.Command(parts[0], parts[1:]...)
 			writer, _ := targetCmd.StdinPipe()
@@ -67,7 +36,7 @@ func main() {
 			out, err := targetCmd.CombinedOutput()
 
 			if err != nil {
-				if writeDebug == true {
+				if config.writeDebug == true {
 					log.Println(targetCmd, err)
 				}
 
@@ -78,13 +47,35 @@ func main() {
 			}
 			w.WriteHeader(200)
 
-			if writeDebug == true {
+			if config.writeDebug == true {
 				os.Stdout.Write(out)
 			}
-
 			w.Write(out)
 		}
-	})
+	}
+}
+
+func main() {
+	osEnv := OsEnv{}
+	readConfig := ReadConfig{}
+	config := readConfig.Read(osEnv)
+
+	if len(config.faasProcess) == 0 {
+		log.Panicln("Provide a valid process via fprocess environmental variable.")
+		return
+	}
+
+	readTimeout := time.Duration(config.readTimeout) * time.Second
+	writeTimeout := time.Duration(config.writeTimeout) * time.Second
+
+	s := &http.Server{
+		Addr:           ":8080",
+		ReadTimeout:    readTimeout,
+		WriteTimeout:   writeTimeout,
+		MaxHeaderBytes: 1 << 20,
+	}
+
+	http.HandleFunc("/", makeRequestHandler(&config))
 
 	log.Fatal(s.ListenAndServe())
 }
diff --git a/watchdog/readconfig.go b/watchdog/readconfig.go
new file mode 100644
index 00000000..56e83bd7
--- /dev/null
+++ b/watchdog/readconfig.go
@@ -0,0 +1,69 @@
+package main
+
+import "strconv"
+
+// HasEnv provides interface for os.Getenv
+type HasEnv interface {
+	Getenv(key string) string
+}
+
+// ReadConfig constitutes config from env variables
+type ReadConfig struct {
+}
+
+func parseBoolValue(val string) bool {
+	if val == "true" {
+		return true
+	}
+	return true
+}
+
+func parseIntValue(val string) int {
+	if len(val) > 0 {
+		parsedVal, parseErr := strconv.Atoi(val)
+
+		if parseErr == nil && parsedVal >= 0 {
+
+			return parsedVal
+		}
+	}
+	return 0
+}
+
+// Read fetches config from environmental variables.
+func (ReadConfig) Read(hasEnv HasEnv) WatchdogConfig {
+	cfg := WatchdogConfig{
+		writeDebug: true,
+	}
+
+	cfg.faasProcess = hasEnv.Getenv("fprocess")
+
+	readTimeout := parseIntValue(hasEnv.Getenv("read_timeout"))
+	writeTimeout := parseIntValue(hasEnv.Getenv("write_timeout"))
+
+	if readTimeout == 0 {
+		cfg.readTimeout = 5
+	} else {
+		cfg.readTimeout = readTimeout
+	}
+	if writeTimeout == 0 {
+		cfg.writeTimeout = 5
+	} else {
+		cfg.writeTimeout = writeTimeout
+	}
+
+	cfg.writeDebug = parseBoolValue(hasEnv.Getenv("write_debug"))
+
+	return cfg
+}
+
+// WatchdogConfig for the process.
+type WatchdogConfig struct {
+	readTimeout  int
+	writeTimeout int
+	// faasProcess is the process to exec
+	faasProcess string
+
+	// writeDebug write console stdout statements to the container
+	writeDebug bool
+}

From 41639cba7e50effa795410607104e780c0f763bf Mon Sep 17 00:00:00 2001
From: Alex <alexellis2@gmail.com>
Date: Tue, 24 Jan 2017 20:06:22 +0000
Subject: [PATCH 27/35] Use time / duration for config

---
 watchdog/config_test.go | 13 ++++++++-----
 watchdog/readconfig.go  | 21 ++++++++++++---------
 2 files changed, 20 insertions(+), 14 deletions(-)

diff --git a/watchdog/config_test.go b/watchdog/config_test.go
index 74f2b025..7291a9ac 100644
--- a/watchdog/config_test.go
+++ b/watchdog/config_test.go
@@ -1,6 +1,9 @@
 package main
 
-import "testing"
+import (
+	"testing"
+	"time"
+)
 
 type EnvBucket struct {
 	Items map[string]string
@@ -62,11 +65,11 @@ func TestRead_EmptyTimeoutConfig(t *testing.T) {
 
 	config := readConfig.Read(defaults)
 
-	if (config.readTimeout) != 5 {
+	if (config.readTimeout) != time.Duration(5)*time.Second {
 		t.Log("readTimeout incorrect")
 		t.Fail()
 	}
-	if (config.writeTimeout) != 5 {
+	if (config.writeTimeout) != time.Duration(5)*time.Second {
 		t.Log("writeTimeout incorrect")
 		t.Fail()
 	}
@@ -80,11 +83,11 @@ func TestRead_ReadAndWriteTimeoutConfig(t *testing.T) {
 	readConfig := ReadConfig{}
 	config := readConfig.Read(defaults)
 
-	if (config.readTimeout) != 10 {
+	if (config.readTimeout) != time.Duration(10)*time.Second {
 		t.Logf("readTimeout incorrect, got: %d\n", config.readTimeout)
 		t.Fail()
 	}
-	if (config.writeTimeout) != 60 {
+	if (config.writeTimeout) != time.Duration(60)*time.Second {
 		t.Logf("writeTimeout incorrect, got: %d\n", config.writeTimeout)
 		t.Fail()
 	}
diff --git a/watchdog/readconfig.go b/watchdog/readconfig.go
index 56e83bd7..102a6e20 100644
--- a/watchdog/readconfig.go
+++ b/watchdog/readconfig.go
@@ -1,6 +1,9 @@
 package main
 
-import "strconv"
+import (
+	"strconv"
+	"time"
+)
 
 // HasEnv provides interface for os.Getenv
 type HasEnv interface {
@@ -42,16 +45,16 @@ func (ReadConfig) Read(hasEnv HasEnv) WatchdogConfig {
 	writeTimeout := parseIntValue(hasEnv.Getenv("write_timeout"))
 
 	if readTimeout == 0 {
-		cfg.readTimeout = 5
-	} else {
-		cfg.readTimeout = readTimeout
+		readTimeout = 5
 	}
+
 	if writeTimeout == 0 {
-		cfg.writeTimeout = 5
-	} else {
-		cfg.writeTimeout = writeTimeout
+		writeTimeout = 5
 	}
 
+	cfg.readTimeout = time.Duration(readTimeout) * time.Second
+	cfg.writeTimeout = time.Duration(writeTimeout) * time.Second
+
 	cfg.writeDebug = parseBoolValue(hasEnv.Getenv("write_debug"))
 
 	return cfg
@@ -59,8 +62,8 @@ func (ReadConfig) Read(hasEnv HasEnv) WatchdogConfig {
 
 // WatchdogConfig for the process.
 type WatchdogConfig struct {
-	readTimeout  int
-	writeTimeout int
+	readTimeout  time.Duration
+	writeTimeout time.Duration
 	// faasProcess is the process to exec
 	faasProcess string
 

From 1079eb9c4255af2584a97a4c612d812573a6327b Mon Sep 17 00:00:00 2001
From: Alex <alexellis2@gmail.com>
Date: Tue, 24 Jan 2017 20:22:28 +0000
Subject: [PATCH 28/35] Add travis and fix Dockerfile for test files

---
 .travis.yml         | 10 ++++++++++
 build.sh            | 20 +++-----------------
 watchdog/Dockerfile |  9 ++++++++-
 3 files changed, 21 insertions(+), 18 deletions(-)
 create mode 100644 .travis.yml

diff --git a/.travis.yml b/.travis.yml
new file mode 100644
index 00000000..23e6904a
--- /dev/null
+++ b/.travis.yml
@@ -0,0 +1,10 @@
+sudo: required
+
+services:
+      - docker
+before_install:
+    - docker pull golang:1.7.3
+
+script:
+    - sh build.sh
+
diff --git a/build.sh b/build.sh
index 2b524325..2b3e6528 100755
--- a/build.sh
+++ b/build.sh
@@ -1,20 +1,6 @@
-#!/bin/sh
+#!/bin/bash
 
-# First do - git clone https://github.com/alexellis/faas && cd faas
+(cd gateway && ./build.sh)
+(cd watchdog && ./build.sh)
 
-docker network create --driver overlay --attachable functions
 
-cd watchdog
-./build.sh
-
-cp ./fwatchdog ../sample-functions/catservice/
-
-docker build --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy \
-    -t alexellis2/faas-catservice .
- #docker service rm catservice ; docker service create --network=functions --name catservice alexellis2/faas-catservice
-
-cd ..
-
-cd gateway
-./build.sh
-# docker rm -f server; docker run -d -v /var/run/docker.sock:/var/run/docker.sock --name server -p 8080:8080 --network=functions alexellis2/faas-gateway
diff --git a/watchdog/Dockerfile b/watchdog/Dockerfile
index ea44643a..d78e326c 100644
--- a/watchdog/Dockerfile
+++ b/watchdog/Dockerfile
@@ -1,6 +1,13 @@
 FROM golang:1.7.3
 RUN mkdir -p /go/src/app
-COPY main.go /go/src/app
+WORKDIR /go/src/app
+COPY main.go        .
+COPY readconfig.go  .
+COPY config_test.go .
+
 WORKDIR /go/src/app
 RUN go get -d -v
+
+RUN go test
 RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o app .
+

From d271649fe5664d831ee50b2cfe73ce737feff068 Mon Sep 17 00:00:00 2001
From: Alex <alexellis2@gmail.com>
Date: Tue, 24 Jan 2017 20:24:42 +0000
Subject: [PATCH 29/35] Add travis logo

---
 README.md | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/README.md b/README.md
index 5e435fca..5702b829 100644
--- a/README.md
+++ b/README.md
@@ -7,6 +7,9 @@ FaaS is a platform for building serverless functions on Docker Swarm Mode with f
 * Each container has a watchdog process that hosts a web server allowing a JSON post request to be forwarded to a desired process via STDIN. The respose is sent to the caller via STDOUT.
 * A gateway provides a view to the containers/functions to the public Internet and collects metrics for Prometheus and in a future version will manage replicas and scale as throughput increases.
 
+[![Build
+Status](https://travis-ci.org/alexellis/faas.svg?branch=master)](https://travis-ci.org/alexellis/faas)
+
 ## Minimum requirements: 
 * Docker 1.13-RC (to support attachable overlay networks)
 * At least a single host in Swarm Mode. (run `docker swarm init`)

From e35fc06745218f0476853132b7dc6efb8fb792df Mon Sep 17 00:00:00 2001
From: Alex <alexellis2@gmail.com>
Date: Tue, 24 Jan 2017 20:26:08 +0000
Subject: [PATCH 30/35] Run build.sh

---
 .travis.yml | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/.travis.yml b/.travis.yml
index 23e6904a..0faacbc9 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -6,5 +6,5 @@ before_install:
     - docker pull golang:1.7.3
 
 script:
-    - sh build.sh
+    - ./build.sh
 

From 34e3d261770aa081458bf7a7233f9dc98fc60da0 Mon Sep 17 00:00:00 2001
From: Alex <alexellis2@gmail.com>
Date: Tue, 24 Jan 2017 20:54:24 +0000
Subject: [PATCH 31/35] Test makehandler

---
 .travis.yml                     |  2 +-
 build.sh                        |  2 -
 watchdog/main.go                |  6 ++-
 watchdog/requesthandler_test.go | 65 +++++++++++++++++++++++++++++++++
 4 files changed, 70 insertions(+), 5 deletions(-)
 create mode 100644 watchdog/requesthandler_test.go

diff --git a/.travis.yml b/.travis.yml
index 0faacbc9..23e6904a 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -6,5 +6,5 @@ before_install:
     - docker pull golang:1.7.3
 
 script:
-    - ./build.sh
+    - sh build.sh
 
diff --git a/build.sh b/build.sh
index 2b3e6528..6f468ccf 100755
--- a/build.sh
+++ b/build.sh
@@ -2,5 +2,3 @@
 
 (cd gateway && ./build.sh)
 (cd watchdog && ./build.sh)
-
-
diff --git a/watchdog/main.go b/watchdog/main.go
index d608a048..07010401 100644
--- a/watchdog/main.go
+++ b/watchdog/main.go
@@ -45,12 +45,14 @@ func makeRequestHandler(config *WatchdogConfig) func(http.ResponseWriter, *http.
 				w.Write(response.Bytes())
 				return
 			}
-			w.WriteHeader(200)
-
 			if config.writeDebug == true {
 				os.Stdout.Write(out)
 			}
+
+			w.WriteHeader(200)
 			w.Write(out)
+		} else {
+			w.WriteHeader(http.StatusMethodNotAllowed)
 		}
 	}
 }
diff --git a/watchdog/requesthandler_test.go b/watchdog/requesthandler_test.go
new file mode 100644
index 00000000..ad72a10c
--- /dev/null
+++ b/watchdog/requesthandler_test.go
@@ -0,0 +1,65 @@
+package main
+
+import (
+	"bytes"
+	"io/ioutil"
+	"net/http"
+	"net/http/httptest"
+	"testing"
+)
+
+func TestHandler_make(t *testing.T) {
+	config := WatchdogConfig{}
+	handler := makeRequestHandler(&config)
+
+	if handler == nil {
+		t.Fail()
+	}
+}
+
+func TestHandler_StatusOKAllowed_ForPOST(t *testing.T) {
+	rr := httptest.NewRecorder()
+
+	body := "hello"
+	req, err := http.NewRequest("POST", "/", bytes.NewBufferString(body))
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	config := WatchdogConfig{
+		faasProcess: "cat",
+	}
+	handler := makeRequestHandler(&config)
+	handler(rr, req)
+
+	required := http.StatusOK
+	if status := rr.Code; status != required {
+		t.Errorf("handler returned wrong status code: got %v, but wanted %v",
+			status, required)
+	}
+
+	buf, _ := ioutil.ReadAll(rr.Body)
+	val := string(buf)
+	if val != body {
+		t.Errorf("Exec of cat did not return input value, %s", val)
+	}
+}
+
+func TestHandler_StatusMethodNotAllowed_ForGet(t *testing.T) {
+	rr := httptest.NewRecorder()
+
+	req, err := http.NewRequest("GET", "/", nil)
+	if err != nil {
+		t.Fatal(err)
+	}
+
+	config := WatchdogConfig{}
+	handler := makeRequestHandler(&config)
+	handler(rr, req)
+
+	required := http.StatusMethodNotAllowed
+	if status := rr.Code; status != required {
+		t.Errorf("handler returned wrong status code: got %v, but wanted %v",
+			status, required)
+	}
+}

From 381556a1e1aea37bff4a704302ae44f15642c11c Mon Sep 17 00:00:00 2001
From: Alex Ellis <alexellis2@gmail.com>
Date: Wed, 25 Jan 2017 19:02:02 +0000
Subject: [PATCH 32/35] Update TestDrive.md

---
 TestDrive.md | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/TestDrive.md b/TestDrive.md
index d82f7fcc..69407d7d 100644
--- a/TestDrive.md
+++ b/TestDrive.md
@@ -18,6 +18,8 @@ This one-shot script clones the code, initialises Docker swarm mode and then dep
 # docker swarm init --advertise-addr=$(ifconfig eth0| grep 'inet addr:'| cut -d: -f2 | awk '{ print $1}') && \
   git clone https://github.com/alexellis/faas && \
   cd faas && \
+  git checkout labels_metrics && \
+  (cd gateway && ./build.sh) && \
   ./deploy_stack.sh && \
   docker service ls
 ```

From 501e813d41d0e6acd7b8c27bfc8f45c44e94c5eb Mon Sep 17 00:00:00 2001
From: Alex Ellis <AlexEllis2@gmail.com>
Date: Wed, 25 Jan 2017 22:29:17 +0000
Subject: [PATCH 33/35] Move handlers into ./handlers

---
 gateway/Dockerfile                      |   1 +
 gateway/Dockerfile.build                |   7 +-
 gateway/handlers/alerthandler.go        |  89 +++++++++++++++
 gateway/handlers/functionshandler.go    |  56 ++++++++++
 gateway/{ => handlers}/proxy.go         |  36 +++---
 gateway/server.go                       | 141 ++----------------------
 gateway/tests/alexhostname_request.json |  24 ++++
 gateway/tests/isalexa_test.go           |  27 +++++
 gateway/tests/unmarshall_test.go        |   5 +-
 9 files changed, 233 insertions(+), 153 deletions(-)
 create mode 100644 gateway/handlers/alerthandler.go
 create mode 100644 gateway/handlers/functionshandler.go
 rename gateway/{ => handlers}/proxy.go (81%)
 create mode 100644 gateway/tests/alexhostname_request.json
 create mode 100644 gateway/tests/isalexa_test.go

diff --git a/gateway/Dockerfile b/gateway/Dockerfile
index d397da87..a0f506c5 100644
--- a/gateway/Dockerfile
+++ b/gateway/Dockerfile
@@ -7,6 +7,7 @@ ENV http_proxy      ""
 ENV https_proxy     ""
 
 COPY gateway    .
+
 COPY assets     assets
 
 CMD ["./gateway"]
diff --git a/gateway/Dockerfile.build b/gateway/Dockerfile.build
index f0b78f6e..7a85b529 100644
--- a/gateway/Dockerfile.build
+++ b/gateway/Dockerfile.build
@@ -6,14 +6,15 @@ RUN go get -d github.com/docker/docker/api/types \
    && go get -d github.com/docker/docker/client \
    && go get github.com/gorilla/mux \
    && go get github.com/prometheus/client_golang/prometheus
-# RUN go get -d github.com/prometheus/client_model/go
+RUN go get -d github.com/Sirupsen/logrus
 
 WORKDIR /go/src/github.com/alexellis/faas/gateway
 
 COPY metrics	metrics
 COPY requests	requests
-COPY tests	    tests
+COPY tests	tests
+COPY handlers	handlers
+
 COPY server.go	.
-COPY proxy.go	.
 
 RUN CGO_ENABLED=0 GOOS=linux go build -a -installsuffix cgo -o app .
diff --git a/gateway/handlers/alerthandler.go b/gateway/handlers/alerthandler.go
new file mode 100644
index 00000000..0017954b
--- /dev/null
+++ b/gateway/handlers/alerthandler.go
@@ -0,0 +1,89 @@
+package handlers
+
+import (
+	"context"
+	"encoding/json"
+	"io/ioutil"
+	"log"
+	"net/http"
+
+	"github.com/alexellis/faas/gateway/requests"
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/client"
+)
+
+func scaleService(req requests.PrometheusAlert, c *client.Client) error {
+	var err error
+	//Todo: convert to loop / handler.
+	serviceName := req.Alerts[0].Labels.FunctionName
+	service, _, inspectErr := c.ServiceInspectWithRaw(context.Background(), serviceName)
+	if inspectErr == nil {
+		var replicas uint64
+
+		if req.Status == "firing" {
+			if *service.Spec.Mode.Replicated.Replicas < 20 {
+				replicas = *service.Spec.Mode.Replicated.Replicas + uint64(5)
+			} else {
+				return err
+			}
+		} else { // Resolved event.
+			// Previously decremented by 5, but event only fires once, so set to 1/1.
+			if *service.Spec.Mode.Replicated.Replicas > 1 {
+				// replicas = *service.Spec.Mode.Replicated.Replicas - uint64(5)
+				// if replicas < 1 {
+				// replicas = 1
+				// }
+				// return nil
+
+				replicas = 1
+			} else {
+				return nil
+			}
+		}
+
+		log.Printf("Scaling %s to %d replicas.\n", serviceName, replicas)
+
+		service.Spec.Mode.Replicated.Replicas = &replicas
+		updateOpts := types.ServiceUpdateOptions{}
+		updateOpts.RegistryAuthFrom = types.RegistryAuthFromSpec
+
+		response, updateErr := c.ServiceUpdate(context.Background(), service.ID, service.Version, service.Spec, updateOpts)
+		if updateErr != nil {
+			err = updateErr
+		}
+		log.Println(response)
+
+	} else {
+		err = inspectErr
+	}
+
+	return err
+}
+
+func MakeAlertHandler(c *client.Client) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		log.Println("Alert received.")
+		body, readErr := ioutil.ReadAll(r.Body)
+		if readErr != nil {
+			log.Println(readErr)
+			return
+		}
+
+		var req requests.PrometheusAlert
+		err := json.Unmarshal(body, &req)
+		if err != nil {
+			log.Println(err)
+			return
+		}
+
+		if len(req.Alerts) > 0 {
+			err := scaleService(req, c)
+			if err != nil {
+				log.Println(err)
+				w.WriteHeader(http.StatusInternalServerError)
+			} else {
+				w.WriteHeader(http.StatusOK)
+			}
+		}
+	}
+}
diff --git a/gateway/handlers/functionshandler.go b/gateway/handlers/functionshandler.go
new file mode 100644
index 00000000..e2c377db
--- /dev/null
+++ b/gateway/handlers/functionshandler.go
@@ -0,0 +1,56 @@
+package handlers
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"net/http"
+
+	"github.com/alexellis/faas/gateway/metrics"
+	"github.com/alexellis/faas/gateway/requests"
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/filters"
+	"github.com/docker/docker/client"
+	io_prometheus_client "github.com/prometheus/client_model/go"
+)
+
+// MakeFunctionReader gives a summary of Function structs with Docker service stats overlaid with Prometheus counters.
+func MakeFunctionReader(metricsOptions metrics.MetricOptions, c *client.Client) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+
+		serviceFilter := filters.NewArgs()
+
+		options := types.ServiceListOptions{
+			Filters: serviceFilter,
+		}
+
+		services, err := c.ServiceList(context.Background(), options)
+		if err != nil {
+			fmt.Println(err)
+		}
+
+		// TODO: Filter only "faas" functions (via metadata?)
+		functions := make([]requests.Function, 0)
+		for _, service := range services {
+			counter, _ := metricsOptions.GatewayFunctionInvocation.GetMetricWithLabelValues(service.Spec.Name)
+
+			// Get the metric's value from ProtoBuf interface (idea via Julius Volz)
+			var protoMetric io_prometheus_client.Metric
+			counter.Write(&protoMetric)
+			invocations := protoMetric.GetCounter().GetValue()
+
+			f := requests.Function{
+				Name:            service.Spec.Name,
+				Image:           service.Spec.TaskTemplate.ContainerSpec.Image,
+				InvocationCount: invocations,
+				Replicas:        *service.Spec.Mode.Replicated.Replicas,
+			}
+			functions = append(functions, f)
+		}
+
+		functionBytes, _ := json.Marshal(functions)
+		w.Header().Set("Content-Type", "application/json")
+		w.WriteHeader(200)
+		w.Write(functionBytes)
+	}
+}
diff --git a/gateway/proxy.go b/gateway/handlers/proxy.go
similarity index 81%
rename from gateway/proxy.go
rename to gateway/handlers/proxy.go
index a426f843..fecb7c2c 100644
--- a/gateway/proxy.go
+++ b/gateway/handlers/proxy.go
@@ -1,4 +1,4 @@
-package main
+package handlers
 
 import (
 	"bytes"
@@ -6,12 +6,12 @@ import (
 	"encoding/json"
 	"fmt"
 	"io/ioutil"
-	"log"
 	"net/http"
 	"strconv"
 	"strings"
 	"time"
 
+	"github.com/Sirupsen/logrus"
 	"github.com/alexellis/faas/gateway/metrics"
 	"github.com/alexellis/faas/gateway/requests"
 	"github.com/docker/docker/api/types"
@@ -20,27 +20,26 @@ import (
 	"github.com/gorilla/mux"
 )
 
-// MakeProxy creates a proxy for HTTP web requests which can be routed to a function.
-func MakeProxy(metrics metrics.MetricOptions, wildcard bool, c *client.Client) http.HandlerFunc {
+// makeProxy creates a proxy for HTTP web requests which can be routed to a function.
+func MakeProxy(metrics metrics.MetricOptions, wildcard bool, c *client.Client, logger *logrus.Logger) http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
 		metrics.GatewayRequestsTotal.Inc()
 
 		if r.Method == "POST" {
-			log.Println(r.Header)
+			logger.Infoln(r.Header)
 			header := r.Header["X-Function"]
-			log.Println(header)
-			// fmt.Println(wildcard)
+			logger.Infoln(header)
 
 			if wildcard == true {
 				vars := mux.Vars(r)
 				name := vars["name"]
 				fmt.Println("invoke by name")
-				lookupInvoke(w, r, metrics, name, c)
+				lookupInvoke(w, r, metrics, name, c, logger)
 			} else if len(header) > 0 {
-				lookupInvoke(w, r, metrics, header[0], c)
+				lookupInvoke(w, r, metrics, header[0], c, logger)
 			} else {
 				requestBody, _ := ioutil.ReadAll(r.Body)
-				alexaService := isAlexa(requestBody)
+				alexaService := IsAlexa(requestBody)
 				fmt.Println(alexaService)
 
 				if len(alexaService.Session.SessionId) > 0 &&
@@ -50,7 +49,7 @@ func MakeProxy(metrics metrics.MetricOptions, wildcard bool, c *client.Client) h
 					fmt.Println("Alexa SDK request found")
 					fmt.Printf("SessionId=%s, Intent=%s, AppId=%s\n", alexaService.Session.SessionId, alexaService.Request.Intent.Name, alexaService.Session.Application.ApplicationId)
 
-					invokeService(w, r, metrics, alexaService.Request.Intent.Name, requestBody)
+					invokeService(w, r, metrics, alexaService.Request.Intent.Name, requestBody, logger)
 				} else {
 					w.WriteHeader(http.StatusBadRequest)
 					w.Write([]byte("Provide an x-function header or a valid Alexa SDK request."))
@@ -60,10 +59,10 @@ func MakeProxy(metrics metrics.MetricOptions, wildcard bool, c *client.Client) h
 	}
 }
 
-func isAlexa(requestBody []byte) requests.AlexaRequestBody {
+func IsAlexa(requestBody []byte) requests.AlexaRequestBody {
 	body := requests.AlexaRequestBody{}
 	buf := bytes.NewBuffer(requestBody)
-	fmt.Println(buf)
+	// fmt.Println(buf)
 	str := buf.String()
 	parts := strings.Split(str, "sessionId")
 	if len(parts) > 1 {
@@ -72,18 +71,18 @@ func isAlexa(requestBody []byte) requests.AlexaRequestBody {
 	return body
 }
 
-func lookupInvoke(w http.ResponseWriter, r *http.Request, metrics metrics.MetricOptions, name string, c *client.Client) {
+func lookupInvoke(w http.ResponseWriter, r *http.Request, metrics metrics.MetricOptions, name string, c *client.Client, logger *logrus.Logger) {
 	exists, err := lookupSwarmService(name, c)
 	if err != nil || exists == false {
 		if err != nil {
-			log.Fatalln(err)
+			logger.Fatalln(err)
 		}
 		w.WriteHeader(http.StatusInternalServerError)
 		w.Write([]byte("Error resolving service."))
 	}
 	if exists == true {
 		requestBody, _ := ioutil.ReadAll(r.Body)
-		invokeService(w, r, metrics, name, requestBody)
+		invokeService(w, r, metrics, name, requestBody, logger)
 	}
 }
 
@@ -96,7 +95,7 @@ func lookupSwarmService(serviceName string, c *client.Client) (bool, error) {
 	return len(services) > 0, err
 }
 
-func invokeService(w http.ResponseWriter, r *http.Request, metrics metrics.MetricOptions, service string, requestBody []byte) {
+func invokeService(w http.ResponseWriter, r *http.Request, metrics metrics.MetricOptions, service string, requestBody []byte, logger *logrus.Logger) {
 	metrics.GatewayFunctionInvocation.WithLabelValues(service).Add(1)
 
 	stamp := strconv.FormatInt(time.Now().Unix(), 10)
@@ -105,9 +104,10 @@ func invokeService(w http.ResponseWriter, r *http.Request, metrics metrics.Metri
 	buf := bytes.NewBuffer(requestBody)
 	url := "http://" + service + ":" + strconv.Itoa(8080) + "/"
 	fmt.Printf("[%s] Forwarding request to: %s\n", stamp, url)
+
 	response, err := http.Post(url, "text/plain", buf)
 	if err != nil {
-		log.Println(err)
+		logger.Infoln(err)
 		w.WriteHeader(500)
 		buf := bytes.NewBufferString("Can't reach service: " + service)
 		w.Write(buf.Bytes())
diff --git a/gateway/server.go b/gateway/server.go
index ef39e65d..2616410c 100644
--- a/gateway/server.go
+++ b/gateway/server.go
@@ -2,140 +2,21 @@ package main
 
 import (
 	"context"
-	"encoding/json"
-	"fmt"
-	"io/ioutil"
 	"log"
 	"net/http"
 	"time"
 
+	"github.com/Sirupsen/logrus"
+	faashandlers "github.com/alexellis/faas/gateway/handlers"
 	"github.com/alexellis/faas/gateway/metrics"
-	"github.com/alexellis/faas/gateway/requests"
-	"github.com/docker/docker/api/types"
-	"github.com/docker/docker/api/types/filters"
 	"github.com/docker/docker/client"
 	"github.com/gorilla/mux"
-	io_prometheus_client "github.com/prometheus/client_model/go"
 )
 
-func scaleService(req requests.PrometheusAlert, c *client.Client) error {
-	var err error
-	//Todo: convert to loop / handler.
-	serviceName := req.Alerts[0].Labels.FunctionName
-	service, _, inspectErr := c.ServiceInspectWithRaw(context.Background(), serviceName)
-	if inspectErr == nil {
-		var replicas uint64
-
-		if req.Status == "firing" {
-			if *service.Spec.Mode.Replicated.Replicas < 20 {
-				replicas = *service.Spec.Mode.Replicated.Replicas + uint64(5)
-			} else {
-				return err
-			}
-		} else { // Resolved event.
-			// Previously decremented by 5, but event only fires once, so set to 1/1.
-			if *service.Spec.Mode.Replicated.Replicas > 1 {
-				// replicas = *service.Spec.Mode.Replicated.Replicas - uint64(5)
-				// if replicas < 1 {
-				// replicas = 1
-				// }
-				// return nil
-
-				replicas = 1
-			} else {
-				return nil
-			}
-		}
-
-		log.Printf("Scaling %s to %d replicas.\n", serviceName, replicas)
-
-		service.Spec.Mode.Replicated.Replicas = &replicas
-		updateOpts := types.ServiceUpdateOptions{}
-		updateOpts.RegistryAuthFrom = types.RegistryAuthFromSpec
-
-		response, updateErr := c.ServiceUpdate(context.Background(), service.ID, service.Version, service.Spec, updateOpts)
-		if updateErr != nil {
-			err = updateErr
-		}
-		log.Println(response)
-
-	} else {
-		err = inspectErr
-	}
-
-	return err
-}
-
-func makeAlertHandler(c *client.Client) http.HandlerFunc {
-	return func(w http.ResponseWriter, r *http.Request) {
-		log.Println("Alert received.")
-		body, readErr := ioutil.ReadAll(r.Body)
-		if readErr != nil {
-			log.Println(readErr)
-			return
-		}
-
-		var req requests.PrometheusAlert
-		err := json.Unmarshal(body, &req)
-		if err != nil {
-			log.Println(err)
-			return
-		}
-
-		if len(req.Alerts) > 0 {
-			err := scaleService(req, c)
-			if err != nil {
-				log.Println(err)
-				w.WriteHeader(http.StatusInternalServerError)
-			} else {
-				w.WriteHeader(http.StatusOK)
-			}
-		}
-	}
-}
-
-// makeFunctionReader gives a summary of Function structs with Docker service stats overlaid with Prometheus counters.
-func makeFunctionReader(metricsOptions metrics.MetricOptions, c *client.Client) http.HandlerFunc {
-	return func(w http.ResponseWriter, r *http.Request) {
-
-		serviceFilter := filters.NewArgs()
-
-		options := types.ServiceListOptions{
-			Filters: serviceFilter,
-		}
-
-		services, err := c.ServiceList(context.Background(), options)
-		if err != nil {
-			fmt.Println(err)
-		}
-
-		// TODO: Filter only "faas" functions (via metadata?)
-		functions := make([]requests.Function, 0)
-		for _, service := range services {
-			counter, _ := metricsOptions.GatewayFunctionInvocation.GetMetricWithLabelValues(service.Spec.Name)
-
-			// Get the metric's value from ProtoBuf interface (idea via Julius Volz)
-			var protoMetric io_prometheus_client.Metric
-			counter.Write(&protoMetric)
-			invocations := protoMetric.GetCounter().GetValue()
-
-			f := requests.Function{
-				Name:            service.Spec.Name,
-				Image:           service.Spec.TaskTemplate.ContainerSpec.Image,
-				InvocationCount: invocations,
-				Replicas:        *service.Spec.Mode.Replicated.Replicas,
-			}
-			functions = append(functions, f)
-		}
-
-		functionBytes, _ := json.Marshal(functions)
-		w.Header().Set("Content-Type", "application/json")
-		w.WriteHeader(200)
-		w.Write(functionBytes)
-	}
-}
-
 func main() {
+	logger := logrus.Logger{}
+	logrus.SetFormatter(&logrus.TextFormatter{})
+
 	var dockerClient *client.Client
 	var err error
 	dockerClient, err = client.NewEnvClient()
@@ -146,16 +27,18 @@ func main() {
 	if err != nil {
 		log.Fatal("Error with Docker server.\n", err)
 	}
-	log.Println("API version: %s, %s\n", dockerVersion.APIVersion, dockerVersion.Version)
+	log.Printf("API version: %s, %s\n", dockerVersion.APIVersion, dockerVersion.Version)
 
 	metricsOptions := metrics.BuildMetricsOptions()
 	metrics.RegisterMetrics(metricsOptions)
 
 	r := mux.NewRouter()
-	r.HandleFunc("/function/{name:[a-zA-Z_]+}", MakeProxy(metricsOptions, true, dockerClient))
-	r.HandleFunc("/system/alert", makeAlertHandler(dockerClient))
-	r.HandleFunc("/system/functions", makeFunctionReader(metricsOptions, dockerClient)).Methods("GET")
-	r.HandleFunc("/", MakeProxy(metricsOptions, false, dockerClient)).Methods("POST")
+	r.HandleFunc("/function/{name:[a-zA-Z_]+}", faashandlers.MakeProxy(metricsOptions, true, dockerClient, &logger))
+
+	r.HandleFunc("/system/alert", faashandlers.MakeAlertHandler(dockerClient))
+	r.HandleFunc("/system/functions", faashandlers.MakeFunctionReader(metricsOptions, dockerClient)).Methods("GET")
+
+	r.HandleFunc("/", faashandlers.MakeProxy(metricsOptions, false, dockerClient, &logger)).Methods("POST")
 
 	metricsHandler := metrics.PrometheusHandler()
 	r.Handle("/metrics", metricsHandler)
diff --git a/gateway/tests/alexhostname_request.json b/gateway/tests/alexhostname_request.json
new file mode 100644
index 00000000..e7953654
--- /dev/null
+++ b/gateway/tests/alexhostname_request.json
@@ -0,0 +1,24 @@
+{
+  "session": {
+    "sessionId": "SessionId.ea96e58d-dc16-43e1-b238-daac4541110c",
+    "application": {
+      "applicationId": "amzn1.ask.skill.72fb1025-aacc-4d05-a582-21344940c023"
+    },
+    "attributes": {},
+    "user": {
+      "userId": "amzn1.ask.account.AEN7KA5DBXAAWQPDUXTXFWBARZ5YZ6TNOQR5CUMV5LCCJTMBZVFP45SZVLGDD5GQBOM7QMELRS7LHG3F2FN2QQQMTBURDL5I4PQ33EHMNNGO4TXWG732Y6SDM2YZKHSPWIIWBH3GSE3Q3TTFAYN2Y66RHBKRANYCNMX2WORMASUGVRHUNBB4HZMJEC7HQDWUSXAOMP77WGJU4AY"
+    },
+    "new": true
+  },
+  "request": {
+    "type": "IntentRequest",
+    "requestId": "EdwRequestId.a934104e-3282-4620-b056-4aa4c5995503",
+    "locale": "en-GB",
+    "timestamp": "2016-12-07T15:50:01Z",
+    "intent": {
+      "name": "HostnameIntent",
+      "slots": {}
+    }
+  },
+  "version": "1.0"
+}
\ No newline at end of file
diff --git a/gateway/tests/isalexa_test.go b/gateway/tests/isalexa_test.go
new file mode 100644
index 00000000..89c86370
--- /dev/null
+++ b/gateway/tests/isalexa_test.go
@@ -0,0 +1,27 @@
+package tests
+
+import (
+	"testing"
+
+	"io/ioutil"
+
+	"github.com/alexellis/faas/gateway/handlers"
+	"github.com/alexellis/faas/gateway/requests"
+)
+
+func TestIsAlexa(t *testing.T) {
+	requestBody, _ := ioutil.ReadFile("./alexhostname_request.json")
+	var result requests.AlexaRequestBody
+
+	result = handlers.IsAlexa(requestBody)
+
+	if len(result.Session.Application.ApplicationId) == 0 {
+		t.Fail()
+	}
+	if len(result.Session.SessionId) == 0 {
+		t.Fail()
+	}
+	if len(result.Request.Intent.Name) == 0 {
+		t.Fail()
+	}
+}
diff --git a/gateway/tests/unmarshall_test.go b/gateway/tests/unmarshall_test.go
index 4fa52288..5a1c675d 100644
--- a/gateway/tests/unmarshall_test.go
+++ b/gateway/tests/unmarshall_test.go
@@ -1,7 +1,6 @@
-package requests
+package tests
 
 import (
-	"fmt"
 	"testing"
 
 	"io/ioutil"
@@ -11,6 +10,7 @@ import (
 	"github.com/alexellis/faas/gateway/requests"
 )
 
+// TestUnmarshallAlert is an exploratory test from TDD'ing the struct to parse a Prometheus alert
 func TestUnmarshallAlert(t *testing.T) {
 	file, _ := ioutil.ReadFile("./test_alert.json")
 
@@ -19,7 +19,6 @@ func TestUnmarshallAlert(t *testing.T) {
 	if err != nil {
 		t.Fatal(err)
 	}
-	fmt.Println("OK", string(file), alert)
 	if (len(alert.Status)) == 0 {
 		t.Fatal("No status read")
 	}

From 1297048799dd6b1ea59affd4e1aa2751e0a7b3a3 Mon Sep 17 00:00:00 2001
From: Traun Leyden <traun.leyden@gmail.com>
Date: Fri, 27 Jan 2017 08:14:57 -0800
Subject: [PATCH 34/35] Fix attempt for reported connection buildup

Regarding https://github.com/golang/go/issues/6785#issuecomment-275669472
---
 watchdog/main.go | 1 +
 1 file changed, 1 insertion(+)

diff --git a/watchdog/main.go b/watchdog/main.go
index 07010401..35e0dd4a 100644
--- a/watchdog/main.go
+++ b/watchdog/main.go
@@ -29,6 +29,7 @@ func makeRequestHandler(config *WatchdogConfig) func(http.ResponseWriter, *http.
 			writer, _ := targetCmd.StdinPipe()
 
 			res, _ := ioutil.ReadAll(r.Body)
+			defer r.Body.Close()
 
 			writer.Write(res)
 			writer.Close()

From 42c0c02950805448d769e41b2c0e8e0365c3ec2d Mon Sep 17 00:00:00 2001
From: Alex <alexellis2@gmail.com>
Date: Fri, 27 Jan 2017 21:07:09 +0000
Subject: [PATCH 35/35] Add integration test and defer

---
 gateway/handlers/proxy.go                | 10 +++
 gateway/tests/integration/routes_test.go | 92 ++++++++++++++++++++++++
 2 files changed, 102 insertions(+)
 create mode 100644 gateway/tests/integration/routes_test.go

diff --git a/gateway/handlers/proxy.go b/gateway/handlers/proxy.go
index fecb7c2c..eb2d5de5 100644
--- a/gateway/handlers/proxy.go
+++ b/gateway/handlers/proxy.go
@@ -35,12 +35,18 @@ func MakeProxy(metrics metrics.MetricOptions, wildcard bool, c *client.Client, l
 				name := vars["name"]
 				fmt.Println("invoke by name")
 				lookupInvoke(w, r, metrics, name, c, logger)
+				defer r.Body.Close()
+
 			} else if len(header) > 0 {
 				lookupInvoke(w, r, metrics, header[0], c, logger)
+				defer r.Body.Close()
+
 			} else {
 				requestBody, _ := ioutil.ReadAll(r.Body)
+				defer r.Body.Close()
 				alexaService := IsAlexa(requestBody)
 				fmt.Println(alexaService)
+				defer r.Body.Close()
 
 				if len(alexaService.Session.SessionId) > 0 &&
 					len(alexaService.Session.Application.ApplicationId) > 0 &&
@@ -50,9 +56,11 @@ func MakeProxy(metrics metrics.MetricOptions, wildcard bool, c *client.Client, l
 					fmt.Printf("SessionId=%s, Intent=%s, AppId=%s\n", alexaService.Session.SessionId, alexaService.Request.Intent.Name, alexaService.Session.Application.ApplicationId)
 
 					invokeService(w, r, metrics, alexaService.Request.Intent.Name, requestBody, logger)
+
 				} else {
 					w.WriteHeader(http.StatusBadRequest)
 					w.Write([]byte("Provide an x-function header or a valid Alexa SDK request."))
+					defer r.Body.Close()
 				}
 			}
 		}
@@ -79,6 +87,7 @@ func lookupInvoke(w http.ResponseWriter, r *http.Request, metrics metrics.Metric
 		}
 		w.WriteHeader(http.StatusInternalServerError)
 		w.Write([]byte("Error resolving service."))
+		defer r.Body.Close()
 	}
 	if exists == true {
 		requestBody, _ := ioutil.ReadAll(r.Body)
@@ -125,6 +134,7 @@ func invokeService(w http.ResponseWriter, r *http.Request, metrics metrics.Metri
 
 	w.WriteHeader(http.StatusOK)
 	w.Write(responseBody)
+
 	seconds := time.Since(start).Seconds()
 	fmt.Printf("[%s] took %f seconds\n", stamp, seconds)
 	metrics.GatewayServerlessServedTotal.Inc()
diff --git a/gateway/tests/integration/routes_test.go b/gateway/tests/integration/routes_test.go
new file mode 100644
index 00000000..a3d68f00
--- /dev/null
+++ b/gateway/tests/integration/routes_test.go
@@ -0,0 +1,92 @@
+package inttests
+
+import (
+	"bytes"
+	"io/ioutil"
+	"log"
+	"net/http"
+	"testing"
+	"time"
+)
+
+// Before running these tests do a Docker stack deploy.
+
+func fireRequest(url string, method string, reqBody string) (string, int, error) {
+	return fireRequestWithHeader(url, method, reqBody, "")
+}
+
+func fireRequestWithHeader(url string, method string, reqBody string, xheader string) (string, int, error) {
+	httpClient := http.Client{
+		Timeout: time.Second * 2, // Maximum of 2 secs
+	}
+
+	req, err := http.NewRequest(method, url, bytes.NewBufferString(reqBody))
+	if err != nil {
+		log.Fatal(err)
+	}
+
+	req.Header.Set("User-Agent", "spacecount-tutorial")
+	if len(xheader) != 0 {
+		req.Header.Set("X-Function", xheader)
+	}
+	res, getErr := httpClient.Do(req)
+	if getErr != nil {
+		log.Fatal(getErr)
+	}
+
+	body, readErr := ioutil.ReadAll(res.Body)
+	defer req.Body.Close()
+	if readErr != nil {
+		log.Fatal(readErr)
+	}
+	return string(body), res.StatusCode, readErr
+}
+
+func Test_Get_Rejected(t *testing.T) {
+	var reqBody string
+	_, code, err := fireRequest("http://localhost:8080/function/func_echoit", http.MethodGet, reqBody)
+	if code != http.StatusInternalServerError {
+		t.Log("Failed")
+	}
+
+	if err != nil {
+		t.Log(err)
+		t.Fail()
+	}
+
+}
+
+func Test_EchoIt_Post_Route_Handler(t *testing.T) {
+	reqBody := "test message"
+	body, code, err := fireRequest("http://localhost:8080/function/func_echoit", http.MethodPost, reqBody)
+
+	if err != nil {
+		t.Log(err)
+		t.Fail()
+	}
+	if code != http.StatusOK {
+		t.Log("Failed")
+	}
+	if body != reqBody {
+		t.Log("Expected body returned")
+		t.Fail()
+	}
+}
+
+func Test_EchoIt_Post_Header_Handler(t *testing.T) {
+	reqBody := "test message"
+	body, code, err := fireRequestWithHeader("http://localhost:8080/", http.MethodPost, reqBody, "func_echoit")
+
+	if err != nil {
+		t.Log(err)
+		t.Fail()
+	}
+	if code != http.StatusOK {
+		t.Log("Failed")
+	}
+	if body != reqBody {
+		t.Log("Expected body returned")
+		t.Fail()
+	}
+
+}