From 7fe67d7af616a97dee058db5e0522759a781a71a Mon Sep 17 00:00:00 2001 From: Simon Pelczer Date: Fri, 23 Mar 2018 22:13:07 +0100 Subject: [PATCH] Implemented the autoscaling steps to be proportions of the max replicas. Introduced an new label to set the scaling factor that is used to calculate th proportions, setting it to 0 also allows to disable scaling. Updated the tests to reflect the changes and added a new test which shows that setting the scaling factor to 0 indeed does disable scaling. Ensured that the scaling factor is always between [0 and 100]. Signed-off-by: Simon Pelczer --- gateway/handlers/alerthandler.go | 18 +++++-- gateway/handlers/alerthandler_test.go | 67 ++++++++++++++++++++------- gateway/handlers/service_query.go | 1 + gateway/plugin/external.go | 22 ++++++++- 4 files changed, 85 insertions(+), 23 deletions(-) diff --git a/gateway/handlers/alerthandler.go b/gateway/handlers/alerthandler.go index 58fd3be9..631b32f3 100644 --- a/gateway/handlers/alerthandler.go +++ b/gateway/handlers/alerthandler.go @@ -14,15 +14,24 @@ import ( "github.com/openfaas/faas/gateway/requests" ) +// DefaultMinReplicas is the minimal amount of replicas for a service. +const DefaultMinReplicas = 1 + // DefaultMaxReplicas is the amount of replicas a service will auto-scale up to. const DefaultMaxReplicas = 20 +// DefaultScalingFactor is the defining proportion for the scaling increments. +const DefaultScalingFactor = 20 + // MinScaleLabel label indicating min scale for a function const MinScaleLabel = "com.openfaas.scale.min" // MaxScaleLabel label indicating max scale for a function const MaxScaleLabel = "com.openfaas.scale.max" +// ScalingFactorLabel label indicates the scaling factor for a function +const ScalingFactorLabel = "com.openfaas.scale.factor" + // MakeAlertHandler handles alerts from Prometheus Alertmanager func MakeAlertHandler(service ServiceQuery) http.HandlerFunc { return func(w http.ResponseWriter, r *http.Request) { @@ -87,7 +96,7 @@ func scaleService(alert requests.PrometheusInnerAlert, service ServiceQuery) err if getErr == nil { status := alert.Status - newReplicas := CalculateReplicas(status, queryResponse.Replicas, uint64(queryResponse.MaxReplicas), queryResponse.MinReplicas) + newReplicas := CalculateReplicas(status, queryResponse.Replicas, uint64(queryResponse.MaxReplicas), queryResponse.MinReplicas, queryResponse.ScalingFactor) log.Printf("[Scale] function=%s %d => %d.\n", serviceName, queryResponse.Replicas, newReplicas) if newReplicas == queryResponse.Replicas { @@ -104,13 +113,12 @@ func scaleService(alert requests.PrometheusInnerAlert, service ServiceQuery) err } // CalculateReplicas decides what replica count to set depending on current/desired amount -func CalculateReplicas(status string, currentReplicas uint64, maxReplicas uint64, minReplicas uint64) uint64 { +func CalculateReplicas(status string, currentReplicas uint64, maxReplicas uint64, minReplicas uint64, scalingFactor uint64) uint64 { newReplicas := currentReplicas - const step = 5 + step := uint64((float64(maxReplicas) / 100) * float64(scalingFactor)) if status == "firing" { if currentReplicas == 1 { - // First jump is from 1 to "step" i.e. 1->5 newReplicas = step } else { if currentReplicas+step > maxReplicas { @@ -123,4 +131,4 @@ func CalculateReplicas(status string, currentReplicas uint64, maxReplicas uint64 newReplicas = minReplicas } return newReplicas -} +} \ No newline at end of file diff --git a/gateway/handlers/alerthandler_test.go b/gateway/handlers/alerthandler_test.go index 8ee71e0e..b74c54d7 100644 --- a/gateway/handlers/alerthandler_test.go +++ b/gateway/handlers/alerthandler_test.go @@ -7,47 +7,82 @@ import ( "testing" ) -func TestScale1to5(t *testing.T) { +func TestDisabledScale(t *testing.T) { minReplicas := uint64(1) - newReplicas := CalculateReplicas("firing", 1, DefaultMaxReplicas, minReplicas) - if newReplicas != 5 { - t.Log("Expected increment in blocks of 5 from 1 to 5") + scalingFactor := uint64(0) + newReplicas := CalculateReplicas("firing", DefaultMinReplicas, DefaultMaxReplicas, minReplicas, scalingFactor) + if newReplicas != 0 { + t.Log("Expected not to scale") t.Fail() } } -func TestScale5to10(t *testing.T) { - minReplicas := uint64(1) - newReplicas := CalculateReplicas("firing", 5, DefaultMaxReplicas, minReplicas) - if newReplicas != 10 { - t.Log("Expected increment in blocks of 5 from 5 to 10") +func TestParameterEdge(t *testing.T){ + minReplicas := uint64(0) + scalingFactor := uint64(0) + newReplicas := CalculateReplicas("firing", DefaultMinReplicas, DefaultMaxReplicas, minReplicas, scalingFactor) + if newReplicas != 0 { + t.Log("Expected not to scale") t.Fail() } } -func TestScaleCeilingOf20Replicas_Noaction(t *testing.T) { +func TestMaxScale(t *testing.T){ minReplicas := uint64(1) - newReplicas := CalculateReplicas("firing", 20, DefaultMaxReplicas, minReplicas) + scalingFactor := uint64(100) + newReplicas := CalculateReplicas("firing", DefaultMinReplicas, DefaultMaxReplicas, minReplicas, scalingFactor) if newReplicas != 20 { t.Log("Expected ceiling of 20 replicas") t.Fail() } } -func TestScaleCeilingOf20Replicas(t *testing.T) { +func TestInitialScale(t *testing.T) { minReplicas := uint64(1) - newReplicas := CalculateReplicas("firing", 19, DefaultMaxReplicas, minReplicas) + scalingFactor := uint64(20) + newReplicas := CalculateReplicas("firing", DefaultMinReplicas, DefaultMaxReplicas, minReplicas, scalingFactor) + if newReplicas != 4 { + t.Log("Expected the increment to equal 4") + t.Fail() + } +} + +func TestScale(t *testing.T) { + minReplicas := uint64(1) + scalingFactor := uint64(20) + newReplicas := CalculateReplicas("firing", 4, DefaultMaxReplicas, minReplicas, scalingFactor) + if newReplicas != 8 { + t.Log("Expected newReplicas to equal 8") + t.Fail() + } +} + +func TestScaleCeiling(t *testing.T) { + minReplicas := uint64(1) + scalingFactor := uint64(20) + newReplicas := CalculateReplicas("firing", 20, DefaultMaxReplicas, minReplicas, scalingFactor) if newReplicas != 20 { t.Log("Expected ceiling of 20 replicas") t.Fail() } } -func TestBackingOff10to1(t *testing.T) { +func TestScaleCeilingEdge(t *testing.T) { minReplicas := uint64(1) - newReplicas := CalculateReplicas("resolved", 10, DefaultMaxReplicas, minReplicas) + scalingFactor := uint64(20) + newReplicas := CalculateReplicas("firing", 19, DefaultMaxReplicas, minReplicas, scalingFactor) + if newReplicas != 20 { + t.Log("Expected ceiling of 20 replicas") + t.Fail() + } +} + +func TestBackingOff(t *testing.T) { + minReplicas := uint64(1) + scalingFactor := uint64(20) + newReplicas := CalculateReplicas("resolved", 8, DefaultMaxReplicas, minReplicas, scalingFactor) if newReplicas != 1 { t.Log("Expected backing off to 1 replica") t.Fail() } -} +} \ No newline at end of file diff --git a/gateway/handlers/service_query.go b/gateway/handlers/service_query.go index c8e34dca..e98fff30 100644 --- a/gateway/handlers/service_query.go +++ b/gateway/handlers/service_query.go @@ -11,5 +11,6 @@ type ServiceQueryResponse struct { Replicas uint64 MaxReplicas uint64 MinReplicas uint64 + ScalingFactor uint64 AvailableReplicas uint64 } diff --git a/gateway/plugin/external.go b/gateway/plugin/external.go index 18398b27..5471e372 100644 --- a/gateway/plugin/external.go +++ b/gateway/plugin/external.go @@ -86,13 +86,15 @@ func (s ExternalServiceQuery) GetReplicas(serviceName string) (handlers.ServiceQ } maxReplicas := uint64(handlers.DefaultMaxReplicas) - minReplicas := uint64(1) + minReplicas := uint64(handlers.DefaultMinReplicas) + scalingFactor := uint64(handlers.DefaultScalingFactor) availableReplicas := function.AvailableReplicas if function.Labels != nil { labels := *function.Labels minScale := labels[handlers.MinScaleLabel] maxScale := labels[handlers.MaxScaleLabel] + scaleFactor := labels[handlers.ScalingFactorLabel] if len(minScale) > 0 { labelValue, err := strconv.Atoi(minScale) @@ -111,12 +113,28 @@ func (s ExternalServiceQuery) GetReplicas(serviceName string) (handlers.ServiceQ maxReplicas = uint64(labelValue) } } + + if len(scaleFactor) > 0 { + labelValue, err := strconv.Atoi(scaleFactor) + if err != nil { + log.Printf("Bad Scaling Factor: %s, should be uint", scaleFactor) + } else { + var temp = uint64(labelValue) + + if temp >= 0 && temp <= 100 { + scalingFactor = temp + } else { + log.Printf("Bad Scaling Factor: %s, is in range [0 - 100]", temp) + } + } + } } return handlers.ServiceQueryResponse{ Replicas: function.Replicas, MaxReplicas: maxReplicas, MinReplicas: minReplicas, + ScalingFactor: scalingFactor, AvailableReplicas: availableReplicas, }, err } @@ -153,4 +171,4 @@ func (s ExternalServiceQuery) SetReplicas(serviceName string, count uint64) erro } return err -} +} \ No newline at end of file