diff --git a/debug.md b/debug.md
new file mode 100644
index 0000000000000000000000000000000000000000..94f3e74abb734df7faf29f42be78a51776d1bad3
--- /dev/null
+++ b/debug.md
@@ -0,0 +1,10 @@
+### Debug information
+
+This is a useful Prometheus query to show:
+
+* Service replicas
+* Rate of invocation
+* Execution time of events
+
+http://localhost:9090/graph?g0.range_input=15m&g0.expr=gateway_service_count&g0.tab=0&g1.range_input=15m&g1.expr=rate(gateway_function_invocation_total%5B20s%5D)&g1.tab=0&g2.range_input=15m&g2.expr=gateway_functions_seconds_sum+%2F+gateway_functions_seconds_count&g2.tab=0
+
diff --git a/gateway/assets/script/bootstrap.js b/gateway/assets/script/bootstrap.js
index 2d0ea18c6354b0008350022400ca468e1240db30..a86b944d44f1922d4663bfa2d677ce5dc9ead058 100644
--- a/gateway/assets/script/bootstrap.js
+++ b/gateway/assets/script/bootstrap.js
@@ -35,7 +35,11 @@ app.controller("home", ['$scope', '$log', '$http', '$location', '$timeout', '$md
$http(options)
.then(function(response) {
- $scope.invocationResponse = response.data;
+ if($scope.invocation && $scope.invocation.contentType == "json") {
+ $scope.invocationResponse = JSON.stringify(response.data, -1, " ");
+ } else {
+ $scope.invocationResponse = response.data;
+ }
$scope.invocationStatus = response.status;
}).catch(function(error1) {
$scope.invocationResponse = error1.statusText + "\n" + error1.data;
diff --git a/gateway/build.sh b/gateway/build.sh
index 6be0d12925bc0b4d95154de31f876faf9ac91d88..9195ed8f643c451429b8acbc3acdb78748358898 100755
--- a/gateway/build.sh
+++ b/gateway/build.sh
@@ -2,12 +2,9 @@
echo Building functions/gateway:build
docker build --build-arg https_proxy=$https_proxy --build-arg http_proxy=$http_proxy \
- -t functions/gateway:build . -f Dockerfile.build
-
-docker create --name gateway_extract functions/gateway:build
-docker cp gateway_extract:/go/src/github.com/alexellis/faas/gateway/app ./gateway
-docker rm -f gateway_extract
-
-echo Building functions/gateway:latest
-
+ -t functions/gateway:build . -f Dockerfile.build && \
+ docker create --name gateway_extract functions/gateway:build && \
+ docker cp gateway_extract:/go/src/github.com/alexellis/faas/gateway/app ./gateway && \
+ docker rm -f gateway_extract && \
+echo Building functions/gateway:latest && \
docker build --no-cache -t functions/gateway:latest-dev .
diff --git a/gateway/handlers/functionshandler.go b/gateway/handlers/functionshandler.go
index 50c0d2895b005651f683c1cc99880dcf71259bff..d493630ae137428091eb8b1c8312fc9133baea96 100644
--- a/gateway/handlers/functionshandler.go
+++ b/gateway/handlers/functionshandler.go
@@ -21,7 +21,9 @@ import (
func getCounterValue(service string, code string, metricsOptions *metrics.MetricOptions) float64 {
- metric, err := metricsOptions.GatewayFunctionInvocation.GetMetricWith(prometheus.Labels{"function_name": service, "code": code})
+ metric, err := metricsOptions.GatewayFunctionInvocation.
+ GetMetricWith(prometheus.Labels{"function_name": service, "code": code})
+
if err != nil {
return 0
}
diff --git a/gateway/handlers/proxy.go b/gateway/handlers/proxy.go
index 1b37c8ebd5e3a87fc24771b3d6f0bbd144dde922..b2e6ecf85f1b8c38503bb4a40d10244a4bc4c3d7 100644
--- a/gateway/handlers/proxy.go
+++ b/gateway/handlers/proxy.go
@@ -21,7 +21,6 @@ import (
// MakeProxy creates a proxy for HTTP web requests which can be routed to a function.
func MakeProxy(metrics metrics.MetricOptions, wildcard bool, c *client.Client, logger *logrus.Logger) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
- metrics.GatewayRequestsTotal.Inc()
if r.Method == "POST" {
logger.Infoln(r.Header)
@@ -55,8 +54,14 @@ func writeHead(service string, metrics metrics.MetricOptions, code int, w http.R
// metrics.GatewayFunctionInvocation.WithLabelValues(service).Add(1)
}
+func trackTime(then time.Time, metrics metrics.MetricOptions, name string) {
+ since := time.Since(then)
+ metrics.GatewayFunctionsHistogram.WithLabelValues(name).Observe(since.Seconds())
+}
+
func lookupInvoke(w http.ResponseWriter, r *http.Request, metrics metrics.MetricOptions, name string, c *client.Client, logger *logrus.Logger) {
exists, err := lookupSwarmService(name, c)
+
if err != nil || exists == false {
if err != nil {
logger.Fatalln(err)
@@ -64,8 +69,11 @@ func lookupInvoke(w http.ResponseWriter, r *http.Request, metrics metrics.Metric
writeHead(name, metrics, http.StatusInternalServerError, w)
w.Write([]byte("Error resolving service."))
defer r.Body.Close()
+ return
}
- if exists == true {
+
+ if exists {
+ defer trackTime(time.Now(), metrics, name)
requestBody, _ := ioutil.ReadAll(r.Body)
invokeService(w, r, metrics, name, requestBody, logger)
}
@@ -81,10 +89,16 @@ func lookupSwarmService(serviceName string, c *client.Client) (bool, error) {
}
func invokeService(w http.ResponseWriter, r *http.Request, metrics metrics.MetricOptions, service string, requestBody []byte, logger *logrus.Logger) {
-
stamp := strconv.FormatInt(time.Now().Unix(), 10)
- start := time.Now()
+ defer func(when time.Time) {
+ seconds := time.Since(when).Seconds()
+
+ fmt.Printf("[%s] took %f seconds\n", stamp, seconds)
+ metrics.GatewayFunctionsHistogram.WithLabelValues(service).Observe(seconds)
+ }(time.Now())
+
+ // start := time.Now()
buf := bytes.NewBuffer(requestBody)
url := "http://" + service + ":" + strconv.Itoa(8080) + "/"
contentType := r.Header.Get("Content-Type")
@@ -118,9 +132,4 @@ func invokeService(w http.ResponseWriter, r *http.Request, metrics metrics.Metri
writeHead(service, metrics, http.StatusOK, w)
w.Write(responseBody)
-
- seconds := time.Since(start).Seconds()
- fmt.Printf("[%s] took %f seconds\n", stamp, seconds)
- metrics.GatewayServerlessServedTotal.Inc()
- metrics.GatewayFunctions.Observe(seconds)
}
diff --git a/gateway/metrics/metrics.go b/gateway/metrics/metrics.go
index 44bba64d02f479ebd1dc1ff3296782f4eb608dc6..ea8503eb48361ded40fb46fe9e4dbcb756d0b7fc 100644
--- a/gateway/metrics/metrics.go
+++ b/gateway/metrics/metrics.go
@@ -8,10 +8,9 @@ import (
// MetricOptions to be used by web handlers
type MetricOptions struct {
- GatewayRequestsTotal prometheus.Counter
- GatewayServerlessServedTotal prometheus.Counter
- GatewayFunctions prometheus.Histogram
- GatewayFunctionInvocation *prometheus.CounterVec
+ GatewayFunctionInvocation *prometheus.CounterVec
+ GatewayFunctionsHistogram *prometheus.HistogramVec
+ ServiceReplicasCounter *prometheus.GaugeVec
}
// PrometheusHandler Bootstraps prometheus for metrics collection
@@ -19,20 +18,14 @@ func PrometheusHandler() http.Handler {
return prometheus.Handler()
}
+// BuildMetricsOptions builds metrics for tracking functions in the API gateway
func BuildMetricsOptions() MetricOptions {
- GatewayRequestsTotal := prometheus.NewCounter(prometheus.CounterOpts{
- Name: "gateway_requests_total",
- Help: "Total amount of HTTP requests to the gateway",
- })
- GatewayServerlessServedTotal := prometheus.NewCounter(prometheus.CounterOpts{
- Name: "gateway_serverless_invocation_total",
- Help: "Total amount of serverless function invocations",
- })
- GatewayFunctions := prometheus.NewHistogram(prometheus.HistogramOpts{
- Name: "gateway_functions",
- Help: "Gateway functions",
- })
- GatewayFunctionInvocation := prometheus.NewCounterVec(
+ gatewayFunctionsHistogram := prometheus.NewHistogramVec(prometheus.HistogramOpts{
+ Name: "gateway_functions_seconds",
+ Help: "Function time taken",
+ }, []string{"function_name"})
+
+ gatewayFunctionInvocation := prometheus.NewCounterVec(
prometheus.CounterOpts{
Name: "gateway_function_invocation_total",
Help: "Individual function metrics",
@@ -40,19 +33,26 @@ func BuildMetricsOptions() MetricOptions {
[]string{"function_name", "code"},
)
+ serviceReplicas := prometheus.NewGaugeVec(
+ prometheus.GaugeOpts{
+ Name: "gateway_service_count",
+ Help: "Docker service replicas",
+ },
+ []string{"function_name"},
+ )
+
metricsOptions := MetricOptions{
- GatewayRequestsTotal: GatewayRequestsTotal,
- GatewayServerlessServedTotal: GatewayServerlessServedTotal,
- GatewayFunctions: GatewayFunctions,
- GatewayFunctionInvocation: GatewayFunctionInvocation,
+ GatewayFunctionsHistogram: gatewayFunctionsHistogram,
+ GatewayFunctionInvocation: gatewayFunctionInvocation,
+ ServiceReplicasCounter: serviceReplicas,
}
return metricsOptions
}
+//RegisterMetrics registers with Prometheus for tracking
func RegisterMetrics(metricsOptions MetricOptions) {
- prometheus.Register(metricsOptions.GatewayRequestsTotal)
- prometheus.Register(metricsOptions.GatewayServerlessServedTotal)
- prometheus.Register(metricsOptions.GatewayFunctions)
prometheus.Register(metricsOptions.GatewayFunctionInvocation)
+ prometheus.Register(metricsOptions.GatewayFunctionsHistogram)
+ prometheus.Register(metricsOptions.ServiceReplicasCounter)
}
diff --git a/gateway/metrics/swarmwatcher.go b/gateway/metrics/swarmwatcher.go
new file mode 100644
index 0000000000000000000000000000000000000000..8d61ecdab16b8fedcbfa9b24e3b194952397b28f
--- /dev/null
+++ b/gateway/metrics/swarmwatcher.go
@@ -0,0 +1,48 @@
+package metrics
+
+import (
+ "context"
+ "fmt"
+ "time"
+
+ "github.com/docker/docker/api/types"
+ "github.com/docker/docker/api/types/filters"
+ "github.com/docker/docker/client"
+)
+
+// AttachSwarmWatcher adds a go-route to monitor the amount of service replicas in the swarm
+// matching a 'function' label.
+func AttachSwarmWatcher(dockerClient *client.Client, metricsOptions MetricOptions) {
+ ticker := time.NewTicker(1 * time.Second)
+ quit := make(chan struct{})
+
+ go func() {
+ for {
+ select {
+ case <-ticker.C:
+ serviceFilter := filters.NewArgs()
+
+ options := types.ServiceListOptions{
+ Filters: serviceFilter,
+ }
+
+ services, err := dockerClient.ServiceList(context.Background(), options)
+ if err != nil {
+ fmt.Println(err)
+ }
+
+ for _, service := range services {
+ if len(service.Spec.TaskTemplate.ContainerSpec.Labels["function"]) > 0 {
+ metricsOptions.ServiceReplicasCounter.
+ WithLabelValues(service.Spec.Name).
+ Set(float64(*service.Spec.Mode.Replicated.Replicas))
+ }
+ }
+ break
+ case <-quit:
+ return
+ }
+ }
+ }()
+
+}
diff --git a/gateway/server.go b/gateway/server.go
index 8f389dd0ea4781f73fe38f46218a36cd3672d4d1..d43b4f5149ea00fcddaf385039b118534262cbd9 100644
--- a/gateway/server.go
+++ b/gateway/server.go
@@ -33,7 +33,7 @@ func main() {
metrics.RegisterMetrics(metricsOptions)
r := mux.NewRouter()
- // r.StrictSlash(false)
+ // r.StrictSlash(false) // This didn't work, so register routes twice.
functionHandler := faasHandlers.MakeProxy(metricsOptions, true, dockerClient, &logger)
r.HandleFunc("/function/{name:[a-zA-Z_0-9]+}", functionHandler)
@@ -47,6 +47,7 @@ func main() {
metricsHandler := metrics.PrometheusHandler()
r.Handle("/metrics", metricsHandler)
+ metrics.AttachSwarmWatcher(dockerClient, metricsOptions)
r.PathPrefix("/").Handler(http.FileServer(http.Dir("./assets/"))).Methods("GET")
s := &http.Server{