diff --git a/gateway/scaling/function_scaler.go b/gateway/scaling/function_scaler.go index 9b37184b35ee6014384f73ceff3e57b40c746d90..1d126701dea6500866335f82f4c64a0a19849b42 100644 --- a/gateway/scaling/function_scaler.go +++ b/gateway/scaling/function_scaler.go @@ -57,12 +57,31 @@ func (f *FunctionScaler) Scale(functionName string) FunctionScaleResult { minReplicas = queryResponse.MinReplicas } - log.Printf("[Scale] function=%s 0 => %d requested", functionName, minReplicas) + scaleResult := backoff(func(attempt int) error { + queryResponse, err := f.Config.ServiceQuery.GetReplicas(functionName) + if err != nil { + return err + } + + f.Cache.Set(functionName, queryResponse) + + if queryResponse.Replicas > 0 { + return nil + } + + log.Printf("[Scale %d] function=%s 0 => %d requested", attempt, functionName, minReplicas) + setScaleErr := f.Config.ServiceQuery.SetReplicas(functionName, minReplicas) + if setScaleErr != nil { + return fmt.Errorf("unable to scale function [%s], err: %s", functionName, setScaleErr) + } - setScaleErr := f.Config.ServiceQuery.SetReplicas(functionName, minReplicas) - if setScaleErr != nil { + return nil + + }, int(f.Config.SetScaleRetries), f.Config.FunctionPollInterval) + + if scaleResult != nil { return FunctionScaleResult{ - Error: fmt.Errorf("unable to scale function [%s], err: %s", functionName, err), + Error: scaleResult, Available: false, Found: true, Duration: time.Since(start), @@ -106,3 +125,23 @@ func (f *FunctionScaler) Scale(functionName string) FunctionScaleResult { Duration: time.Since(start), } } + +type routine func(attempt int) error + +func backoff(r routine, attempts int, interval time.Duration) error { + var err error + + for i := 0; i < attempts; i++ { + res := r(i) + if res != nil { + err = res + + log.Printf("Attempt: %d, had error: %s\n", i, res) + } else { + err = nil + break + } + time.Sleep(interval) + } + return err +} diff --git a/gateway/scaling/scaling_config.go b/gateway/scaling/scaling_config.go index 918647a87e065ef3b931a58d9515c7081364df29..d6eb55c1c2f9643db2ad627e87472d9de69795bb 100644 --- a/gateway/scaling/scaling_config.go +++ b/gateway/scaling/scaling_config.go @@ -9,7 +9,8 @@ type ScalingConfig struct { // MaxPollCount attempts to query a function before giving up MaxPollCount uint - // FunctionPollInterval delay or interval between polling a function's readiness status + // FunctionPollInterval delay or interval between polling a function's + // readiness status FunctionPollInterval time.Duration // CacheExpiry life-time for a cache entry before considering invalid @@ -17,4 +18,8 @@ type ScalingConfig struct { // ServiceQuery queries available/ready replicas for function ServiceQuery ServiceQuery + + // SetScaleRetries is the number of times to try scaling a function before + // giving up due to errors + SetScaleRetries uint } diff --git a/gateway/server.go b/gateway/server.go index 97fe4925ae055a28931b6fa82a42d08892fa1e90..69555941c3144fa8e0fd55a323261cb205d2d5cb 100644 --- a/gateway/server.go +++ b/gateway/server.go @@ -137,7 +137,8 @@ func main() { if config.ScaleFromZero { scalingConfig := scaling.ScalingConfig{ MaxPollCount: uint(1000), - FunctionPollInterval: time.Millisecond * 10, + SetScaleRetries: uint(20), + FunctionPollInterval: time.Millisecond * 50, CacheExpiry: time.Second * 5, // freshness of replica values before going stale ServiceQuery: alertHandler, }