mirror of
https://github.com/openfaas/faas.git
synced 2025-06-09 16:56:47 +00:00
Enable backoff/retries on scaling up
- this change is needed for Docker Swarm which may give an error when several concurrent requests come in to scale a deployment. Tested on Docker Swarm before/after with the hey tool and figlet scaled down to zero replicas. Signed-off-by: Alex Ellis (VMware) <alexellis2@gmail.com>
This commit is contained in:
parent
446c8672ae
commit
117707df14
@ -57,12 +57,31 @@ func (f *FunctionScaler) Scale(functionName string) FunctionScaleResult {
|
|||||||
minReplicas = queryResponse.MinReplicas
|
minReplicas = queryResponse.MinReplicas
|
||||||
}
|
}
|
||||||
|
|
||||||
log.Printf("[Scale] function=%s 0 => %d requested", functionName, minReplicas)
|
scaleResult := backoff(func(attempt int) error {
|
||||||
|
queryResponse, err := f.Config.ServiceQuery.GetReplicas(functionName)
|
||||||
|
if err != nil {
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
|
||||||
setScaleErr := f.Config.ServiceQuery.SetReplicas(functionName, minReplicas)
|
f.Cache.Set(functionName, queryResponse)
|
||||||
if setScaleErr != nil {
|
|
||||||
|
if queryResponse.Replicas > 0 {
|
||||||
|
return nil
|
||||||
|
}
|
||||||
|
|
||||||
|
log.Printf("[Scale %d] function=%s 0 => %d requested", attempt, functionName, minReplicas)
|
||||||
|
setScaleErr := f.Config.ServiceQuery.SetReplicas(functionName, minReplicas)
|
||||||
|
if setScaleErr != nil {
|
||||||
|
return fmt.Errorf("unable to scale function [%s], err: %s", functionName, setScaleErr)
|
||||||
|
}
|
||||||
|
|
||||||
|
return nil
|
||||||
|
|
||||||
|
}, int(f.Config.SetScaleRetries), f.Config.FunctionPollInterval)
|
||||||
|
|
||||||
|
if scaleResult != nil {
|
||||||
return FunctionScaleResult{
|
return FunctionScaleResult{
|
||||||
Error: fmt.Errorf("unable to scale function [%s], err: %s", functionName, err),
|
Error: scaleResult,
|
||||||
Available: false,
|
Available: false,
|
||||||
Found: true,
|
Found: true,
|
||||||
Duration: time.Since(start),
|
Duration: time.Since(start),
|
||||||
@ -106,3 +125,23 @@ func (f *FunctionScaler) Scale(functionName string) FunctionScaleResult {
|
|||||||
Duration: time.Since(start),
|
Duration: time.Since(start),
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
type routine func(attempt int) error
|
||||||
|
|
||||||
|
func backoff(r routine, attempts int, interval time.Duration) error {
|
||||||
|
var err error
|
||||||
|
|
||||||
|
for i := 0; i < attempts; i++ {
|
||||||
|
res := r(i)
|
||||||
|
if res != nil {
|
||||||
|
err = res
|
||||||
|
|
||||||
|
log.Printf("Attempt: %d, had error: %s\n", i, res)
|
||||||
|
} else {
|
||||||
|
err = nil
|
||||||
|
break
|
||||||
|
}
|
||||||
|
time.Sleep(interval)
|
||||||
|
}
|
||||||
|
return err
|
||||||
|
}
|
||||||
|
@ -9,7 +9,8 @@ type ScalingConfig struct {
|
|||||||
// MaxPollCount attempts to query a function before giving up
|
// MaxPollCount attempts to query a function before giving up
|
||||||
MaxPollCount uint
|
MaxPollCount uint
|
||||||
|
|
||||||
// FunctionPollInterval delay or interval between polling a function's readiness status
|
// FunctionPollInterval delay or interval between polling a function's
|
||||||
|
// readiness status
|
||||||
FunctionPollInterval time.Duration
|
FunctionPollInterval time.Duration
|
||||||
|
|
||||||
// CacheExpiry life-time for a cache entry before considering invalid
|
// CacheExpiry life-time for a cache entry before considering invalid
|
||||||
@ -17,4 +18,8 @@ type ScalingConfig struct {
|
|||||||
|
|
||||||
// ServiceQuery queries available/ready replicas for function
|
// ServiceQuery queries available/ready replicas for function
|
||||||
ServiceQuery ServiceQuery
|
ServiceQuery ServiceQuery
|
||||||
|
|
||||||
|
// SetScaleRetries is the number of times to try scaling a function before
|
||||||
|
// giving up due to errors
|
||||||
|
SetScaleRetries uint
|
||||||
}
|
}
|
||||||
|
@ -137,7 +137,8 @@ func main() {
|
|||||||
if config.ScaleFromZero {
|
if config.ScaleFromZero {
|
||||||
scalingConfig := scaling.ScalingConfig{
|
scalingConfig := scaling.ScalingConfig{
|
||||||
MaxPollCount: uint(1000),
|
MaxPollCount: uint(1000),
|
||||||
FunctionPollInterval: time.Millisecond * 10,
|
SetScaleRetries: uint(20),
|
||||||
|
FunctionPollInterval: time.Millisecond * 50,
|
||||||
CacheExpiry: time.Second * 5, // freshness of replica values before going stale
|
CacheExpiry: time.Second * 5, // freshness of replica values before going stale
|
||||||
ServiceQuery: alertHandler,
|
ServiceQuery: alertHandler,
|
||||||
}
|
}
|
||||||
|
Loading…
x
Reference in New Issue
Block a user