Simplify instrumentation metrics.

- fix javascript deserialization in UI with json response
-  create swarm watcher to monitor replicas

debug.md

@@ -0,0 +1,10 @@
+### Debug information
+
+This is a useful Prometheus query to show:
+
+* Service replicas
+* Rate of invocation
+* Execution time of events
+
+http://localhost:9090/graph?g0.range_input=15m&g0.expr=gateway_service_count&g0.tab=0&g1.range_input=15m&g1.expr=rate(gateway_function_invocation_total%5B20s%5D)&g1.tab=0&g2.range_input=15m&g2.expr=gateway_functions_seconds_sum+%2F+gateway_functions_seconds_count&g2.tab=0
+

gateway/assets/script/bootstrap.js

@@ -35,7 +35,11 @@ app.controller("home", ['$scope', '$log', '$http', '$location', '$timeout', '$md
 
         $http(options)
             .then(function(response) {
-                $scope.invocationResponse = response.data;
+                if($scope.invocation && $scope.invocation.contentType == "json") {
+                    $scope.invocationResponse = JSON.stringify(response.data, -1, "  ");
+                } else {
+                    $scope.invocationResponse = response.data;
+                }
                 $scope.invocationStatus = response.status;
             }).catch(function(error1) {
                 $scope.invocationResponse = error1.statusText + "\n" + error1.data;

gateway/build.sh

@@ -2,12 +2,9 @@
 echo Building functions/gateway:build
 
 docker build --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy \
-    -t functions/gateway:build . -f Dockerfile.build
-
-docker create --name gateway_extract functions/gateway:build 
-docker cp gateway_extract:/go/src/github.com/alexellis/faas/gateway/app ./gateway
-docker rm -f gateway_extract
-
-echo Building functions/gateway:latest
-
+    -t functions/gateway:build . -f Dockerfile.build && \
+  docker create --name gateway_extract functions/gateway:build  && \
+  docker cp gateway_extract:/go/src/github.com/alexellis/faas/gateway/app ./gateway && \
+  docker rm -f gateway_extract && \
+echo Building functions/gateway:latest && \
 docker build --no-cache -t functions/gateway:latest-dev .

gateway/handlers/functionshandler.go

@@ -21,7 +21,9 @@ import (
 
 func getCounterValue(service string, code string, metricsOptions *metrics.MetricOptions) float64 {
 
-	metric, err := metricsOptions.GatewayFunctionInvocation.GetMetricWith(prometheus.Labels{"function_name": service, "code": code})
+	metric, err := metricsOptions.GatewayFunctionInvocation.
+		GetMetricWith(prometheus.Labels{"function_name": service, "code": code})
+
 	if err != nil {
 		return 0
 	}

gateway/handlers/proxy.go

@@ -21,7 +21,6 @@ import (
 // MakeProxy creates a proxy for HTTP web requests which can be routed to a function.
 func MakeProxy(metrics metrics.MetricOptions, wildcard bool, c *client.Client, logger *logrus.Logger) http.HandlerFunc {
 	return func(w http.ResponseWriter, r *http.Request) {
-		metrics.GatewayRequestsTotal.Inc()
 
 		if r.Method == "POST" {
 			logger.Infoln(r.Header)
@@ -55,8 +54,14 @@ func writeHead(service string, metrics metrics.MetricOptions, code int, w http.R
 	// metrics.GatewayFunctionInvocation.WithLabelValues(service).Add(1)
 }
 
+func trackTime(then time.Time, metrics metrics.MetricOptions, name string) {
+	since := time.Since(then)
+	metrics.GatewayFunctionsHistogram.WithLabelValues(name).Observe(since.Seconds())
+}
+
 func lookupInvoke(w http.ResponseWriter, r *http.Request, metrics metrics.MetricOptions, name string, c *client.Client, logger *logrus.Logger) {
 	exists, err := lookupSwarmService(name, c)
+
 	if err != nil || exists == false {
 		if err != nil {
 			logger.Fatalln(err)
@@ -64,8 +69,11 @@ func lookupInvoke(w http.ResponseWriter, r *http.Request, metrics metrics.Metric
 		writeHead(name, metrics, http.StatusInternalServerError, w)
 		w.Write([]byte("Error resolving service."))
 		defer r.Body.Close()
+		return
 	}
-	if exists == true {
+
+	if exists {
+		defer trackTime(time.Now(), metrics, name)
 		requestBody, _ := ioutil.ReadAll(r.Body)
 		invokeService(w, r, metrics, name, requestBody, logger)
 	}
@@ -81,10 +89,16 @@ func lookupSwarmService(serviceName string, c *client.Client) (bool, error) {
 }
 
 func invokeService(w http.ResponseWriter, r *http.Request, metrics metrics.MetricOptions, service string, requestBody []byte, logger *logrus.Logger) {
-
 	stamp := strconv.FormatInt(time.Now().Unix(), 10)
 
-	start := time.Now()
+	defer func(when time.Time) {
+		seconds := time.Since(when).Seconds()
+
+		fmt.Printf("[%s] took %f seconds\n", stamp, seconds)
+		metrics.GatewayFunctionsHistogram.WithLabelValues(service).Observe(seconds)
+	}(time.Now())
+
+	// start := time.Now()
 	buf := bytes.NewBuffer(requestBody)
 	url := "http://" + service + ":" + strconv.Itoa(8080) + "/"
 	contentType := r.Header.Get("Content-Type")
@@ -118,9 +132,4 @@ func invokeService(w http.ResponseWriter, r *http.Request, metrics metrics.Metri
 
 	writeHead(service, metrics, http.StatusOK, w)
 	w.Write(responseBody)
-
-	seconds := time.Since(start).Seconds()
-	fmt.Printf("[%s] took %f seconds\n", stamp, seconds)
-	metrics.GatewayServerlessServedTotal.Inc()
-	metrics.GatewayFunctions.Observe(seconds)
 }

gateway/metrics/metrics.go

@@ -8,10 +8,9 @@ import (
 
 // MetricOptions to be used by web handlers
 type MetricOptions struct {
-	GatewayRequestsTotal         prometheus.Counter
-	GatewayServerlessServedTotal prometheus.Counter
-	GatewayFunctions             prometheus.Histogram
-	GatewayFunctionInvocation    *prometheus.CounterVec
+	GatewayFunctionInvocation *prometheus.CounterVec
+	GatewayFunctionsHistogram *prometheus.HistogramVec
+	ServiceReplicasCounter    *prometheus.GaugeVec
 }
 
 // PrometheusHandler Bootstraps prometheus for metrics collection
@@ -19,20 +18,14 @@ func PrometheusHandler() http.Handler {
 	return prometheus.Handler()
 }
 
+// BuildMetricsOptions builds metrics for tracking functions in the API gateway
 func BuildMetricsOptions() MetricOptions {
-	GatewayRequestsTotal := prometheus.NewCounter(prometheus.CounterOpts{
-		Name: "gateway_requests_total",
-		Help: "Total amount of HTTP requests to the gateway",
-	})
-	GatewayServerlessServedTotal := prometheus.NewCounter(prometheus.CounterOpts{
-		Name: "gateway_serverless_invocation_total",
-		Help: "Total amount of serverless function invocations",
-	})
-	GatewayFunctions := prometheus.NewHistogram(prometheus.HistogramOpts{
-		Name: "gateway_functions",
-		Help: "Gateway functions",
-	})
-	GatewayFunctionInvocation := prometheus.NewCounterVec(
+	gatewayFunctionsHistogram := prometheus.NewHistogramVec(prometheus.HistogramOpts{
+		Name: "gateway_functions_seconds",
+		Help: "Function time taken",
+	}, []string{"function_name"})
+
+	gatewayFunctionInvocation := prometheus.NewCounterVec(
 		prometheus.CounterOpts{
 			Name: "gateway_function_invocation_total",
 			Help: "Individual function metrics",
@@ -40,19 +33,26 @@ func BuildMetricsOptions() MetricOptions {
 		[]string{"function_name", "code"},
 	)
 
+	serviceReplicas := prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: "gateway_service_count",
+			Help: "Docker service replicas",
+		},
+		[]string{"function_name"},
+	)
+
 	metricsOptions := MetricOptions{
-		GatewayRequestsTotal:         GatewayRequestsTotal,
-		GatewayServerlessServedTotal: GatewayServerlessServedTotal,
-		GatewayFunctions:             GatewayFunctions,
-		GatewayFunctionInvocation:    GatewayFunctionInvocation,
+		GatewayFunctionsHistogram: gatewayFunctionsHistogram,
+		GatewayFunctionInvocation: gatewayFunctionInvocation,
+		ServiceReplicasCounter:    serviceReplicas,
 	}
 
 	return metricsOptions
 }
 
+//RegisterMetrics registers with Prometheus for tracking
 func RegisterMetrics(metricsOptions MetricOptions) {
-	prometheus.Register(metricsOptions.GatewayRequestsTotal)
-	prometheus.Register(metricsOptions.GatewayServerlessServedTotal)
-	prometheus.Register(metricsOptions.GatewayFunctions)
 	prometheus.Register(metricsOptions.GatewayFunctionInvocation)
+	prometheus.Register(metricsOptions.GatewayFunctionsHistogram)
+	prometheus.Register(metricsOptions.ServiceReplicasCounter)
 }

gateway/metrics/swarmwatcher.go

@@ -0,0 +1,48 @@
+package metrics
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/api/types/filters"
+	"github.com/docker/docker/client"
+)
+
+// AttachSwarmWatcher adds a go-route to monitor the amount of service replicas in the swarm
+// matching a 'function' label.
+func AttachSwarmWatcher(dockerClient *client.Client, metricsOptions MetricOptions) {
+	ticker := time.NewTicker(1 * time.Second)
+	quit := make(chan struct{})
+
+	go func() {
+		for {
+			select {
+			case <-ticker.C:
+				serviceFilter := filters.NewArgs()
+
+				options := types.ServiceListOptions{
+					Filters: serviceFilter,
+				}
+
+				services, err := dockerClient.ServiceList(context.Background(), options)
+				if err != nil {
+					fmt.Println(err)
+				}
+
+				for _, service := range services {
+					if len(service.Spec.TaskTemplate.ContainerSpec.Labels["function"]) > 0 {
+						metricsOptions.ServiceReplicasCounter.
+							WithLabelValues(service.Spec.Name).
+							Set(float64(*service.Spec.Mode.Replicated.Replicas))
+					}
+				}
+				break
+			case <-quit:
+				return
+			}
+		}
+	}()
+
+}

gateway/server.go

@@ -33,7 +33,7 @@ func main() {
 	metrics.RegisterMetrics(metricsOptions)
 
 	r := mux.NewRouter()
-	// r.StrictSlash(false)
+	// r.StrictSlash(false)	// This didn't work, so register routes twice.
 
 	functionHandler := faasHandlers.MakeProxy(metricsOptions, true, dockerClient, &logger)
 	r.HandleFunc("/function/{name:[a-zA-Z_0-9]+}", functionHandler)
@@ -47,6 +47,7 @@ func main() {
 
 	metricsHandler := metrics.PrometheusHandler()
 	r.Handle("/metrics", metricsHandler)
+	metrics.AttachSwarmWatcher(dockerClient, metricsOptions)
 
 	r.PathPrefix("/").Handler(http.FileServer(http.Dir("./assets/"))).Methods("GET")
 	s := &http.Server{