From 96812d2cd8afcd1c6e73cdab8ab1761ea83f56ad Mon Sep 17 00:00:00 2001 From: "Alex Ellis (OpenFaaS Ltd)" Date: Thu, 19 Dec 2019 10:18:28 +0000 Subject: [PATCH] Record metrics for invocations when they start * This experimental patch records metrics as invocations start so that the metrics can be used to make better scale to zero decisions in faas-idler. Tested with Kubernetes on a single-node cluster, metrics reported as expected. Existing metrics still report. Signed-off-by: Alex Ellis (OpenFaaS Ltd) --- gateway/Dockerfile | 2 +- gateway/Dockerfile.arm64 | 2 +- gateway/Dockerfile.armhf | 2 +- gateway/handlers/forwarding_proxy.go | 7 +++- gateway/handlers/notifier_handler.go | 2 +- gateway/handlers/notifier_wrapper_test.go | 2 +- gateway/handlers/notifiers.go | 39 ++++++++++++-------- gateway/{server.go => main.go} | 0 gateway/metrics/exporter.go | 3 ++ gateway/metrics/metrics.go | 12 ++++++ gateway/vendor/github.com/gorilla/mux/go.mod | 2 + 11 files changed, 51 insertions(+), 22 deletions(-) rename gateway/{server.go => main.go} (100%) diff --git a/gateway/Dockerfile b/gateway/Dockerfile index cd3d5d34..f68b3868 100644 --- a/gateway/Dockerfile +++ b/gateway/Dockerfile @@ -28,7 +28,7 @@ COPY queue queue COPY plugin plugin COPY version version COPY scaling scaling -COPY server.go . +COPY main.go . # Run a gofmt and exclude all vendored code. RUN license-check -path ./ --verbose=false "Alex Ellis" "OpenFaaS Authors" "OpenFaaS Author(s)" \ diff --git a/gateway/Dockerfile.arm64 b/gateway/Dockerfile.arm64 index 515dbf1f..2ee06808 100644 --- a/gateway/Dockerfile.arm64 +++ b/gateway/Dockerfile.arm64 @@ -25,7 +25,7 @@ COPY queue queue COPY plugin plugin COPY version version COPY scaling scaling -COPY server.go . +COPY main.go . # Run a gofmt and exclude all vendored code. RUN license-check -path ./ --verbose=false "Alex Ellis" "OpenFaaS Project" "OpenFaaS Authors" "OpenFaaS Author(s)" \ diff --git a/gateway/Dockerfile.armhf b/gateway/Dockerfile.armhf index 578f6ca9..0a5da0ea 100644 --- a/gateway/Dockerfile.armhf +++ b/gateway/Dockerfile.armhf @@ -25,7 +25,7 @@ COPY queue queue COPY plugin plugin COPY version version COPY scaling scaling -COPY server.go . +COPY main.go . # Run a gofmt and exclude all vendored code. RUN license-check -path ./ --verbose=false "Alex Ellis" "OpenFaaS Project" "OpenFaaS Authors" "OpenFaaS Author(s)" \ diff --git a/gateway/handlers/forwarding_proxy.go b/gateway/handlers/forwarding_proxy.go index f270f59e..02d5ce53 100644 --- a/gateway/handlers/forwarding_proxy.go +++ b/gateway/handlers/forwarding_proxy.go @@ -53,9 +53,12 @@ func MakeForwardingProxyHandler(proxy *types.HTTPClientReverseProxy, return func(w http.ResponseWriter, r *http.Request) { baseURL := baseURLResolver.Resolve(r) originalURL := r.URL.String() - requestURL := urlPathTransformer.Transform(r) + for _, notifier := range notifiers { + notifier.Notify(r.Method, requestURL, originalURL, http.StatusProcessing, "started", time.Second*0) + } + start := time.Now() statusCode, err := forwardRequest(w, r, proxy.Client, baseURL, requestURL, proxy.Timeout, writeRequestURI, serviceAuthInjector) @@ -66,7 +69,7 @@ func MakeForwardingProxyHandler(proxy *types.HTTPClientReverseProxy, } for _, notifier := range notifiers { - notifier.Notify(r.Method, requestURL, originalURL, statusCode, seconds) + notifier.Notify(r.Method, requestURL, originalURL, statusCode, "completed", seconds) } } } diff --git a/gateway/handlers/notifier_handler.go b/gateway/handlers/notifier_handler.go index c7ca6c26..8c9f66f2 100644 --- a/gateway/handlers/notifier_handler.go +++ b/gateway/handlers/notifier_handler.go @@ -18,7 +18,7 @@ func MakeNotifierWrapper(next http.HandlerFunc, notifiers []HTTPNotifier) http.H url := r.URL.String() for _, notifier := range notifiers { - notifier.Notify(r.Method, url, url, writer.Status(), time.Since(then)) + notifier.Notify(r.Method, url, url, writer.Status(), "completed", time.Since(then)) } } } diff --git a/gateway/handlers/notifier_wrapper_test.go b/gateway/handlers/notifier_wrapper_test.go index 39f29424..fee77e8b 100644 --- a/gateway/handlers/notifier_wrapper_test.go +++ b/gateway/handlers/notifier_wrapper_test.go @@ -84,6 +84,6 @@ type testNotifier struct { } // Notify about service metrics -func (tf *testNotifier) Notify(method string, URL string, originalURL string, statusCode int, duration time.Duration) { +func (tf *testNotifier) Notify(method string, URL string, originalURL string, statusCode int, event string, duration time.Duration) { tf.StatusReceived = statusCode } diff --git a/gateway/handlers/notifiers.go b/gateway/handlers/notifiers.go index 0398b291..3bf0ae69 100644 --- a/gateway/handlers/notifiers.go +++ b/gateway/handlers/notifiers.go @@ -13,7 +13,7 @@ import ( // HTTPNotifier notify about HTTP request/response type HTTPNotifier interface { - Notify(method string, URL string, originalURL string, statusCode int, duration time.Duration) + Notify(method string, URL string, originalURL string, statusCode int, event string, duration time.Duration) } // PrometheusServiceNotifier notifier for core service endpoints @@ -22,7 +22,7 @@ type PrometheusServiceNotifier struct { } // Notify about service metrics -func (psn PrometheusServiceNotifier) Notify(method string, URL string, originalURL string, statusCode int, duration time.Duration) { +func (psn PrometheusServiceNotifier) Notify(method string, URL string, originalURL string, statusCode int, event string, duration time.Duration) { code := fmt.Sprintf("%d", statusCode) path := urlToLabel(URL) @@ -46,19 +46,26 @@ type PrometheusFunctionNotifier struct { } // Notify records metrics in Prometheus -func (p PrometheusFunctionNotifier) Notify(method string, URL string, originalURL string, statusCode int, duration time.Duration) { - seconds := duration.Seconds() - serviceName := getServiceName(originalURL) +func (p PrometheusFunctionNotifier) Notify(method string, URL string, originalURL string, statusCode int, event string, duration time.Duration) { + if event == "completed" { - p.Metrics.GatewayFunctionsHistogram. - WithLabelValues(serviceName). - Observe(seconds) + seconds := duration.Seconds() + serviceName := getServiceName(originalURL) - code := strconv.Itoa(statusCode) + p.Metrics.GatewayFunctionsHistogram. + WithLabelValues(serviceName). + Observe(seconds) + + code := strconv.Itoa(statusCode) + + p.Metrics.GatewayFunctionInvocation. + With(prometheus.Labels{"function_name": serviceName, "code": code}). + Inc() + } else if event == "started" { + serviceName := getServiceName(originalURL) + p.Metrics.StartedCounter.WithLabelValues(serviceName).Inc() + } - p.Metrics.GatewayFunctionInvocation. - With(prometheus.Labels{"function_name": serviceName, "code": code}). - Inc() } func getServiceName(urlValue string) string { @@ -83,7 +90,9 @@ func getServiceName(urlValue string) string { type LoggingNotifier struct { } -// Notify a log about a request -func (LoggingNotifier) Notify(method string, URL string, originalURL string, statusCode int, duration time.Duration) { - log.Printf("Forwarded [%s] to %s - [%d] - %fs", method, originalURL, statusCode, duration.Seconds()) +// Notify the LoggingNotifier about a request +func (LoggingNotifier) Notify(method string, URL string, originalURL string, statusCode int, event string, duration time.Duration) { + if event == "completed" { + log.Printf("Forwarded [%s] to %s - [%d] - %fs seconds", method, originalURL, statusCode, duration.Seconds()) + } } diff --git a/gateway/server.go b/gateway/main.go similarity index 100% rename from gateway/server.go rename to gateway/main.go diff --git a/gateway/metrics/exporter.go b/gateway/metrics/exporter.go index 365cca88..8adbfab8 100644 --- a/gateway/metrics/exporter.go +++ b/gateway/metrics/exporter.go @@ -41,6 +41,7 @@ func (e *Exporter) Describe(ch chan<- *prometheus.Desc) { e.metricOptions.GatewayFunctionInvocation.Describe(ch) e.metricOptions.GatewayFunctionsHistogram.Describe(ch) e.metricOptions.ServiceReplicasGauge.Describe(ch) + e.metricOptions.StartedCounter.Describe(ch) e.metricOptions.ServiceMetrics.Counter.Describe(ch) e.metricOptions.ServiceMetrics.Histogram.Describe(ch) @@ -51,6 +52,8 @@ func (e *Exporter) Collect(ch chan<- prometheus.Metric) { e.metricOptions.GatewayFunctionInvocation.Collect(ch) e.metricOptions.GatewayFunctionsHistogram.Collect(ch) + e.metricOptions.StartedCounter.Collect(ch) + e.metricOptions.ServiceReplicasGauge.Reset() for _, service := range e.services { e.metricOptions.ServiceReplicasGauge. diff --git a/gateway/metrics/metrics.go b/gateway/metrics/metrics.go index 1bb79658..fc5a763a 100644 --- a/gateway/metrics/metrics.go +++ b/gateway/metrics/metrics.go @@ -17,6 +17,7 @@ type MetricOptions struct { GatewayFunctionsHistogram *prometheus.HistogramVec ServiceReplicasGauge *prometheus.GaugeVec ServiceMetrics *ServiceMetricOptions + StartedCounter *prometheus.CounterVec } // ServiceMetricOptions provides RED metrics @@ -81,6 +82,16 @@ func BuildMetricsOptions() MetricOptions { []string{"method", "path", "status"}, ) + startedCounter := prometheus.NewCounterVec( + prometheus.CounterOpts{ + Namespace: "gateway", + Subsystem: "function", + Name: "invocation_started", + Help: "The total number of function HTTP requests started.", + }, + []string{"function_name"}, + ) + serviceMetricOptions := &ServiceMetricOptions{ Counter: counter, Histogram: histogram, @@ -91,6 +102,7 @@ func BuildMetricsOptions() MetricOptions { GatewayFunctionInvocation: gatewayFunctionInvocation, ServiceReplicasGauge: serviceReplicas, ServiceMetrics: serviceMetricOptions, + StartedCounter: startedCounter, } return metricsOptions diff --git a/gateway/vendor/github.com/gorilla/mux/go.mod b/gateway/vendor/github.com/gorilla/mux/go.mod index cfc8ede5..ba9be08f 100644 --- a/gateway/vendor/github.com/gorilla/mux/go.mod +++ b/gateway/vendor/github.com/gorilla/mux/go.mod @@ -1 +1,3 @@ module github.com/gorilla/mux + +go 1.13