Implemented the autoscaling steps to be proportions of the max replicas.

Introduced an new label to set the scaling factor that is used to calculate th proportions, setting it to 0 also allows to disable scaling.
Updated the tests to reflect the changes and added a new test which shows that setting the scaling factor to 0 indeed does disable scaling.
Ensured that the scaling factor is always between [0 and 100].

Signed-off-by: Simon Pelczer <templum.dev@gmail.com>
This commit is contained in:
Simon Pelczer 2018-03-23 22:13:07 +01:00 committed by Alex Ellis
parent a3cbe0b4a4
commit 7fe67d7af6
4 changed files with 85 additions and 23 deletions

View File

@ -14,15 +14,24 @@ import (
"github.com/openfaas/faas/gateway/requests" "github.com/openfaas/faas/gateway/requests"
) )
// DefaultMinReplicas is the minimal amount of replicas for a service.
const DefaultMinReplicas = 1
// DefaultMaxReplicas is the amount of replicas a service will auto-scale up to. // DefaultMaxReplicas is the amount of replicas a service will auto-scale up to.
const DefaultMaxReplicas = 20 const DefaultMaxReplicas = 20
// DefaultScalingFactor is the defining proportion for the scaling increments.
const DefaultScalingFactor = 20
// MinScaleLabel label indicating min scale for a function // MinScaleLabel label indicating min scale for a function
const MinScaleLabel = "com.openfaas.scale.min" const MinScaleLabel = "com.openfaas.scale.min"
// MaxScaleLabel label indicating max scale for a function // MaxScaleLabel label indicating max scale for a function
const MaxScaleLabel = "com.openfaas.scale.max" const MaxScaleLabel = "com.openfaas.scale.max"
// ScalingFactorLabel label indicates the scaling factor for a function
const ScalingFactorLabel = "com.openfaas.scale.factor"
// MakeAlertHandler handles alerts from Prometheus Alertmanager // MakeAlertHandler handles alerts from Prometheus Alertmanager
func MakeAlertHandler(service ServiceQuery) http.HandlerFunc { func MakeAlertHandler(service ServiceQuery) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) { return func(w http.ResponseWriter, r *http.Request) {
@ -87,7 +96,7 @@ func scaleService(alert requests.PrometheusInnerAlert, service ServiceQuery) err
if getErr == nil { if getErr == nil {
status := alert.Status status := alert.Status
newReplicas := CalculateReplicas(status, queryResponse.Replicas, uint64(queryResponse.MaxReplicas), queryResponse.MinReplicas) newReplicas := CalculateReplicas(status, queryResponse.Replicas, uint64(queryResponse.MaxReplicas), queryResponse.MinReplicas, queryResponse.ScalingFactor)
log.Printf("[Scale] function=%s %d => %d.\n", serviceName, queryResponse.Replicas, newReplicas) log.Printf("[Scale] function=%s %d => %d.\n", serviceName, queryResponse.Replicas, newReplicas)
if newReplicas == queryResponse.Replicas { if newReplicas == queryResponse.Replicas {
@ -104,13 +113,12 @@ func scaleService(alert requests.PrometheusInnerAlert, service ServiceQuery) err
} }
// CalculateReplicas decides what replica count to set depending on current/desired amount // CalculateReplicas decides what replica count to set depending on current/desired amount
func CalculateReplicas(status string, currentReplicas uint64, maxReplicas uint64, minReplicas uint64) uint64 { func CalculateReplicas(status string, currentReplicas uint64, maxReplicas uint64, minReplicas uint64, scalingFactor uint64) uint64 {
newReplicas := currentReplicas newReplicas := currentReplicas
const step = 5 step := uint64((float64(maxReplicas) / 100) * float64(scalingFactor))
if status == "firing" { if status == "firing" {
if currentReplicas == 1 { if currentReplicas == 1 {
// First jump is from 1 to "step" i.e. 1->5
newReplicas = step newReplicas = step
} else { } else {
if currentReplicas+step > maxReplicas { if currentReplicas+step > maxReplicas {

View File

@ -7,45 +7,80 @@ import (
"testing" "testing"
) )
func TestScale1to5(t *testing.T) { func TestDisabledScale(t *testing.T) {
minReplicas := uint64(1) minReplicas := uint64(1)
newReplicas := CalculateReplicas("firing", 1, DefaultMaxReplicas, minReplicas) scalingFactor := uint64(0)
if newReplicas != 5 { newReplicas := CalculateReplicas("firing", DefaultMinReplicas, DefaultMaxReplicas, minReplicas, scalingFactor)
t.Log("Expected increment in blocks of 5 from 1 to 5") if newReplicas != 0 {
t.Log("Expected not to scale")
t.Fail() t.Fail()
} }
} }
func TestScale5to10(t *testing.T) { func TestParameterEdge(t *testing.T){
minReplicas := uint64(1) minReplicas := uint64(0)
newReplicas := CalculateReplicas("firing", 5, DefaultMaxReplicas, minReplicas) scalingFactor := uint64(0)
if newReplicas != 10 { newReplicas := CalculateReplicas("firing", DefaultMinReplicas, DefaultMaxReplicas, minReplicas, scalingFactor)
t.Log("Expected increment in blocks of 5 from 5 to 10") if newReplicas != 0 {
t.Log("Expected not to scale")
t.Fail() t.Fail()
} }
} }
func TestScaleCeilingOf20Replicas_Noaction(t *testing.T) { func TestMaxScale(t *testing.T){
minReplicas := uint64(1) minReplicas := uint64(1)
newReplicas := CalculateReplicas("firing", 20, DefaultMaxReplicas, minReplicas) scalingFactor := uint64(100)
newReplicas := CalculateReplicas("firing", DefaultMinReplicas, DefaultMaxReplicas, minReplicas, scalingFactor)
if newReplicas != 20 { if newReplicas != 20 {
t.Log("Expected ceiling of 20 replicas") t.Log("Expected ceiling of 20 replicas")
t.Fail() t.Fail()
} }
} }
func TestScaleCeilingOf20Replicas(t *testing.T) { func TestInitialScale(t *testing.T) {
minReplicas := uint64(1) minReplicas := uint64(1)
newReplicas := CalculateReplicas("firing", 19, DefaultMaxReplicas, minReplicas) scalingFactor := uint64(20)
newReplicas := CalculateReplicas("firing", DefaultMinReplicas, DefaultMaxReplicas, minReplicas, scalingFactor)
if newReplicas != 4 {
t.Log("Expected the increment to equal 4")
t.Fail()
}
}
func TestScale(t *testing.T) {
minReplicas := uint64(1)
scalingFactor := uint64(20)
newReplicas := CalculateReplicas("firing", 4, DefaultMaxReplicas, minReplicas, scalingFactor)
if newReplicas != 8 {
t.Log("Expected newReplicas to equal 8")
t.Fail()
}
}
func TestScaleCeiling(t *testing.T) {
minReplicas := uint64(1)
scalingFactor := uint64(20)
newReplicas := CalculateReplicas("firing", 20, DefaultMaxReplicas, minReplicas, scalingFactor)
if newReplicas != 20 { if newReplicas != 20 {
t.Log("Expected ceiling of 20 replicas") t.Log("Expected ceiling of 20 replicas")
t.Fail() t.Fail()
} }
} }
func TestBackingOff10to1(t *testing.T) { func TestScaleCeilingEdge(t *testing.T) {
minReplicas := uint64(1) minReplicas := uint64(1)
newReplicas := CalculateReplicas("resolved", 10, DefaultMaxReplicas, minReplicas) scalingFactor := uint64(20)
newReplicas := CalculateReplicas("firing", 19, DefaultMaxReplicas, minReplicas, scalingFactor)
if newReplicas != 20 {
t.Log("Expected ceiling of 20 replicas")
t.Fail()
}
}
func TestBackingOff(t *testing.T) {
minReplicas := uint64(1)
scalingFactor := uint64(20)
newReplicas := CalculateReplicas("resolved", 8, DefaultMaxReplicas, minReplicas, scalingFactor)
if newReplicas != 1 { if newReplicas != 1 {
t.Log("Expected backing off to 1 replica") t.Log("Expected backing off to 1 replica")
t.Fail() t.Fail()

View File

@ -11,5 +11,6 @@ type ServiceQueryResponse struct {
Replicas uint64 Replicas uint64
MaxReplicas uint64 MaxReplicas uint64
MinReplicas uint64 MinReplicas uint64
ScalingFactor uint64
AvailableReplicas uint64 AvailableReplicas uint64
} }

View File

@ -86,13 +86,15 @@ func (s ExternalServiceQuery) GetReplicas(serviceName string) (handlers.ServiceQ
} }
maxReplicas := uint64(handlers.DefaultMaxReplicas) maxReplicas := uint64(handlers.DefaultMaxReplicas)
minReplicas := uint64(1) minReplicas := uint64(handlers.DefaultMinReplicas)
scalingFactor := uint64(handlers.DefaultScalingFactor)
availableReplicas := function.AvailableReplicas availableReplicas := function.AvailableReplicas
if function.Labels != nil { if function.Labels != nil {
labels := *function.Labels labels := *function.Labels
minScale := labels[handlers.MinScaleLabel] minScale := labels[handlers.MinScaleLabel]
maxScale := labels[handlers.MaxScaleLabel] maxScale := labels[handlers.MaxScaleLabel]
scaleFactor := labels[handlers.ScalingFactorLabel]
if len(minScale) > 0 { if len(minScale) > 0 {
labelValue, err := strconv.Atoi(minScale) labelValue, err := strconv.Atoi(minScale)
@ -111,12 +113,28 @@ func (s ExternalServiceQuery) GetReplicas(serviceName string) (handlers.ServiceQ
maxReplicas = uint64(labelValue) maxReplicas = uint64(labelValue)
} }
} }
if len(scaleFactor) > 0 {
labelValue, err := strconv.Atoi(scaleFactor)
if err != nil {
log.Printf("Bad Scaling Factor: %s, should be uint", scaleFactor)
} else {
var temp = uint64(labelValue)
if temp >= 0 && temp <= 100 {
scalingFactor = temp
} else {
log.Printf("Bad Scaling Factor: %s, is in range [0 - 100]", temp)
}
}
}
} }
return handlers.ServiceQueryResponse{ return handlers.ServiceQueryResponse{
Replicas: function.Replicas, Replicas: function.Replicas,
MaxReplicas: maxReplicas, MaxReplicas: maxReplicas,
MinReplicas: minReplicas, MinReplicas: minReplicas,
ScalingFactor: scalingFactor,
AvailableReplicas: availableReplicas, AvailableReplicas: availableReplicas,
}, err }, err
} }