From f4fde50ca80636276cdf62115535a637df518fb3 Mon Sep 17 00:00:00 2001
From: Alex Ellis <AlexEllis2@gmail.com>
Date: Sun, 22 Jan 2017 20:00:48 +0000
Subject: [PATCH] Add +5 / -5 Replica scaling in response to AlertManager

---
 gateway/proxy.go       |  2 +-
 gateway/server.go      | 45 ++++++++++++++++++++++++++++++++++++++----
 prometheus/alert.rules |  2 +-
 3 files changed, 43 insertions(+), 6 deletions(-)

diff --git a/gateway/proxy.go b/gateway/proxy.go
index 1eab535f..a426f843 100644
--- a/gateway/proxy.go
+++ b/gateway/proxy.go
@@ -29,7 +29,7 @@ func MakeProxy(metrics metrics.MetricOptions, wildcard bool, c *client.Client) h
 			log.Println(r.Header)
 			header := r.Header["X-Function"]
 			log.Println(header)
-			fmt.Println(wildcard)
+			// fmt.Println(wildcard)
 
 			if wildcard == true {
 				vars := mux.Vars(r)
diff --git a/gateway/server.go b/gateway/server.go
index 44e96169..0443f96e 100644
--- a/gateway/server.go
+++ b/gateway/server.go
@@ -10,6 +10,7 @@ import (
 	"time"
 
 	"github.com/alexellis/faas/gateway/metrics"
+	"github.com/alexellis/faas/gateway/requests"
 	"github.com/docker/docker/api/types"
 	"github.com/docker/docker/api/types/filters"
 	"github.com/docker/docker/client"
@@ -17,14 +18,50 @@ import (
 	io_prometheus_client "github.com/prometheus/client_model/go"
 )
 
-func makeAlertHandler() http.HandlerFunc {
+func makeAlertHandler(c *client.Client) http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
-		fmt.Println("Alert received.")
+		log.Println("Alert received.")
 		body, _ := ioutil.ReadAll(r.Body)
 		fmt.Println(string(body))
 		// Todo: parse alert, validate alert and scale up or down function
 
-		w.WriteHeader(http.StatusOK)
+		var req requests.PrometheusAlert
+		err := json.Unmarshal(body, &req)
+		if err != nil {
+			log.Println(err)
+		}
+
+		if len(req.Alerts) > 0 {
+			serviceName := req.Alerts[0].Labels.FunctionName
+			service, _, _ := c.ServiceInspectWithRaw(context.Background(), serviceName)
+			var replicas uint64
+
+			if req.Status == "firing" {
+				if *service.Spec.Mode.Replicated.Replicas < 20 {
+					replicas = *service.Spec.Mode.Replicated.Replicas + uint64(5)
+				} else {
+					return
+				}
+			} else {
+				replicas = *service.Spec.Mode.Replicated.Replicas - uint64(5)
+				if replicas <= 0 {
+					replicas = 1
+				}
+			}
+			log.Printf("Scaling %s to %d replicas.\n", serviceName, replicas)
+
+			service.Spec.Mode.Replicated.Replicas = &replicas
+			updateOpts := types.ServiceUpdateOptions{}
+			updateOpts.RegistryAuthFrom = types.RegistryAuthFromSpec
+
+			response, updateErr := c.ServiceUpdate(context.Background(), service.ID, service.Version, service.Spec, updateOpts)
+			if updateErr != nil {
+				w.WriteHeader(http.StatusInternalServerError)
+				log.Println(response)
+			}
+
+			w.WriteHeader(http.StatusOK)
+		}
 	}
 }
 
@@ -89,7 +126,7 @@ func main() {
 
 	r := mux.NewRouter()
 	r.HandleFunc("/function/{name:[a-zA-Z_]+}", MakeProxy(metricsOptions, true, dockerClient))
-	r.HandleFunc("/system/alert", makeAlertHandler())
+	r.HandleFunc("/system/alert", makeAlertHandler(dockerClient))
 	r.HandleFunc("/system/functions", makeFunctionReader(metricsOptions, dockerClient)).Methods("GET")
 	r.HandleFunc("/", MakeProxy(metricsOptions, false, dockerClient)).Methods("POST")
 
diff --git a/prometheus/alert.rules b/prometheus/alert.rules
index 68e7ce42..05785ddc 100644
--- a/prometheus/alert.rules
+++ b/prometheus/alert.rules
@@ -3,7 +3,7 @@ ALERT service_down
 
 ALERT APIHighInvocationRate
   IF rate ( gateway_function_invocation_total [10s] ) > 5
-  FOR 30s
+  FOR 5s
   LABELS {
     service = "gateway",
     severity = "major",