Add namespace in function name for metrics

This commit adds namespace in function names while logging metrics to
prometheus, irrespective of the function is invoked with namespace suffix
or not.

This is also required to add multiple namespace support to faas-idler

https://github.com/openfaas-incubator/faas-idler/issues/37 which is part
of https://github.com/openfaas/faas-netes/issues/511

Signed-off-by: Vivek Singh <vivekkmr45@yahoo.in>

gateway/handlers/notifiers.go

@@ -43,15 +43,21 @@ func urlToLabel(path string) string {
 // PrometheusFunctionNotifier records metrics to Prometheus
 type PrometheusFunctionNotifier struct {
 	Metrics *metrics.MetricOptions
+	//FunctionNamespace default namespace of the function
+	FunctionNamespace string
 }
 
 // Notify records metrics in Prometheus
 func (p PrometheusFunctionNotifier) Notify(method string, URL string, originalURL string, statusCode int, event string, duration time.Duration) {
+	serviceName := getServiceName(originalURL)
+	if len(p.FunctionNamespace) > 0 {
+		if index := strings.Index(serviceName, "."); index == -1 {
+			serviceName = fmt.Sprintf("%s.%s", serviceName, p.FunctionNamespace)
+		}
+	}
+
 	if event == "completed" {
-
 		seconds := duration.Seconds()
-		serviceName := getServiceName(originalURL)
-
 		p.Metrics.GatewayFunctionsHistogram.
 			WithLabelValues(serviceName).
 			Observe(seconds)
@@ -62,7 +68,6 @@ func (p PrometheusFunctionNotifier) Notify(method string, URL string, originalUR
 			With(prometheus.Labels{"function_name": serviceName, "code": code}).
 			Inc()
 	} else if event == "started" {
-		serviceName := getServiceName(originalURL)
 		p.Metrics.GatewayFunctionInvocationStarted.WithLabelValues(serviceName).Inc()
 	}
 

gateway/main.go

@@ -77,7 +77,8 @@ func main() {
 	loggingNotifier := handlers.LoggingNotifier{}
 
 	prometheusNotifier := handlers.PrometheusFunctionNotifier{
-		Metrics: &metricsOptions,
+		Metrics:           &metricsOptions,
+		FunctionNamespace: config.Namespace,
 	}
 
 	prometheusServiceNotifier := handlers.PrometheusServiceNotifier{

gateway/metrics/exporter.go

@@ -6,10 +6,12 @@ package metrics
 
 import (
 	"encoding/json"
+	"fmt"
 	"io/ioutil"
 	"net"
 	"net/http"
 	"net/url"
+	"path"
 	"time"
 
 	"log"
@@ -56,8 +58,14 @@ func (e *Exporter) Collect(ch chan<- prometheus.Metric) {
 
 	e.metricOptions.ServiceReplicasGauge.Reset()
 	for _, service := range e.services {
+		var serviceName string
+		if len(service.Namespace) > 0 {
+			serviceName = fmt.Sprintf("%s.%s", service.Name, service.Namespace)
+		} else {
+			serviceName = service.Name
+		}
 		e.metricOptions.ServiceReplicasGauge.
-			WithLabelValues(service.Name).
+			WithLabelValues(serviceName).
 			Set(float64(service.Replicas))
 	}
 
@@ -72,9 +80,46 @@ func (e *Exporter) StartServiceWatcher(endpointURL url.URL, metricsOptions Metri
 	ticker := time.NewTicker(interval)
 	quit := make(chan struct{})
 
-	timeout := 3 * time.Second
+	go func() {
+		for {
+			select {
+			case <-ticker.C:
 
-	proxyClient := http.Client{
+				namespaces, err := e.getNamespaces(endpointURL)
+				if err != nil {
+					log.Println(err)
+				}
+
+				if len(namespaces) == 0 {
+					emptyNamespace := ""
+					services, err := e.getFunctions(endpointURL, emptyNamespace)
+					if err != nil {
+						log.Println(err)
+						continue
+					}
+					e.services = services
+				} else {
+					for _, namespace := range namespaces {
+						services, err := e.getFunctions(endpointURL, namespace)
+						if err != nil {
+							log.Println(err)
+							continue
+						}
+						e.services = append(e.services, services...)
+					}
+				}
+
+				break
+			case <-quit:
+				return
+			}
+		}
+	}()
+}
+
+func (e *Exporter) getHTTPClient(timeout time.Duration) http.Client {
+
+	return http.Client{
 		Transport: &http.Transport{
 			Proxy: http.ProxyFromEnvironment,
 			DialContext: (&net.Dialer{
@@ -87,45 +132,70 @@ func (e *Exporter) StartServiceWatcher(endpointURL url.URL, metricsOptions Metri
 			ExpectContinueTimeout: 1500 * time.Millisecond,
 		},
 	}
-
-	go func() {
-		for {
-			select {
-			case <-ticker.C:
-
-				get, err := http.NewRequest(http.MethodGet, endpointURL.String()+"system/functions", nil)
-				if err != nil {
-					log.Println(err)
-					return
-				}
-
-				if e.credentials != nil {
-					get.SetBasicAuth(e.credentials.User, e.credentials.Password)
-				}
-
-				services := []types.FunctionStatus{}
-				res, err := proxyClient.Do(get)
-				if err != nil {
-					log.Println(err)
-					continue
-				}
-				bytesOut, readErr := ioutil.ReadAll(res.Body)
-				if readErr != nil {
-					log.Println(err)
-					continue
-				}
-				unmarshalErr := json.Unmarshal(bytesOut, &services)
-				if unmarshalErr != nil {
-					log.Println(err)
-					continue
-				}
-
-				e.services = services
-
-				break
-			case <-quit:
-				return
-			}
-		}
-	}()
+}
+
+func (e *Exporter) getFunctions(endpointURL url.URL, namespace string) ([]types.FunctionStatus, error) {
+	timeout := 3 * time.Second
+	proxyClient := e.getHTTPClient(timeout)
+
+	endpointURL.Path = path.Join(endpointURL.Path, "/system/functions")
+	if len(namespace) > 0 {
+		q := endpointURL.Query()
+		q.Set("namespace", namespace)
+		endpointURL.RawQuery = q.Encode()
+	}
+
+	get, _ := http.NewRequest(http.MethodGet, endpointURL.String(), nil)
+	if e.credentials != nil {
+		get.SetBasicAuth(e.credentials.User, e.credentials.Password)
+	}
+
+	services := []types.FunctionStatus{}
+	res, err := proxyClient.Do(get)
+	if err != nil {
+		return services, err
+	}
+
+	bytesOut, readErr := ioutil.ReadAll(res.Body)
+	if readErr != nil {
+		return services, readErr
+	}
+
+	unmarshalErr := json.Unmarshal(bytesOut, &services)
+	if unmarshalErr != nil {
+		return services, unmarshalErr
+	}
+	return services, nil
+}
+
+func (e *Exporter) getNamespaces(endpointURL url.URL) ([]string, error) {
+	namespaces := []string{}
+
+	get, _ := http.NewRequest(http.MethodGet, endpointURL.String()+"system/namespaces", nil)
+	if e.credentials != nil {
+		get.SetBasicAuth(e.credentials.User, e.credentials.Password)
+	}
+
+	timeout := 3 * time.Second
+	proxyClient := e.getHTTPClient(timeout)
+
+	res, err := proxyClient.Do(get)
+	if err != nil {
+		return namespaces, err
+	}
+
+	if res.StatusCode == http.StatusNotFound {
+		return namespaces, nil
+	}
+
+	bytesOut, readErr := ioutil.ReadAll(res.Body)
+	if readErr != nil {
+		return namespaces, readErr
+	}
+
+	unmarshalErr := json.Unmarshal(bytesOut, &namespaces)
+	if unmarshalErr != nil {
+		return namespaces, unmarshalErr
+	}
+	return namespaces, nil
 }