From 38713184e59d0f4902925043142c682ee5d72547 Mon Sep 17 00:00:00 2001
From: Alex <alexellis2@gmail.com>
Date: Mon, 13 Mar 2017 22:57:05 +0000
Subject: [PATCH] Add status code instrumentation for functions
---
gateway/handlers/functionshandler.go | 24 +++++++++++++++++-------
gateway/handlers/proxy.go | 19 ++++++++++++++-----
gateway/metrics/metrics.go | 2 +-
3 files changed, 32 insertions(+), 13 deletions(-)
diff --git a/gateway/handlers/functionshandler.go b/gateway/handlers/functionshandler.go
index d62743f1..50c0d289 100644
--- a/gateway/handlers/functionshandler.go
+++ b/gateway/handlers/functionshandler.go
@@ -15,9 +15,24 @@ import (
"github.com/docker/docker/api/types/filters"
"github.com/docker/docker/api/types/swarm"
"github.com/docker/docker/client"
+ "github.com/prometheus/client_golang/prometheus"
io_prometheus_client "github.com/prometheus/client_model/go"
)
+func getCounterValue(service string, code string, metricsOptions *metrics.MetricOptions) float64 {
+
+ metric, err := metricsOptions.GatewayFunctionInvocation.GetMetricWith(prometheus.Labels{"function_name": service, "code": code})
+ if err != nil {
+ return 0
+ }
+
+ // Get the metric's value from ProtoBuf interface (idea via Julius Volz)
+ var protoMetric io_prometheus_client.Metric
+ metric.Write(&protoMetric)
+ invocations := protoMetric.GetCounter().GetValue()
+ return invocations
+}
+
// MakeFunctionReader gives a summary of Function structs with Docker service stats overlaid with Prometheus counters.
func MakeFunctionReader(metricsOptions metrics.MetricOptions, c *client.Client) http.HandlerFunc {
return func(w http.ResponseWriter, r *http.Request) {
@@ -39,13 +54,8 @@ func MakeFunctionReader(metricsOptions metrics.MetricOptions, c *client.Client)
for _, service := range services {
if len(service.Spec.TaskTemplate.ContainerSpec.Labels["function"]) > 0 {
-
- counter, _ := metricsOptions.GatewayFunctionInvocation.GetMetricWithLabelValues(service.Spec.Name)
-
- // Get the metric's value from ProtoBuf interface (idea via Julius Volz)
- var protoMetric io_prometheus_client.Metric
- counter.Write(&protoMetric)
- invocations := protoMetric.GetCounter().GetValue()
+ invocations := getCounterValue(service.Spec.Name, "200", &metricsOptions) +
+ getCounterValue(service.Spec.Name, "500", &metricsOptions)
f := requests.Function{
Name: service.Spec.Name,
diff --git a/gateway/handlers/proxy.go b/gateway/handlers/proxy.go
index dcf4de09..1b37c8eb 100644
--- a/gateway/handlers/proxy.go
+++ b/gateway/handlers/proxy.go
@@ -15,6 +15,7 @@ import (
"github.com/docker/docker/api/types/filters"
"github.com/docker/docker/client"
"github.com/gorilla/mux"
+ "github.com/prometheus/client_golang/prometheus"
)
// MakeProxy creates a proxy for HTTP web requests which can be routed to a function.
@@ -46,13 +47,21 @@ func MakeProxy(metrics metrics.MetricOptions, wildcard bool, c *client.Client, l
}
}
+func writeHead(service string, metrics metrics.MetricOptions, code int, w http.ResponseWriter) {
+ w.WriteHeader(code)
+
+ metrics.GatewayFunctionInvocation.With(prometheus.Labels{"function_name": service, "code": strconv.Itoa(code)}).Inc()
+
+ // metrics.GatewayFunctionInvocation.WithLabelValues(service).Add(1)
+}
+
func lookupInvoke(w http.ResponseWriter, r *http.Request, metrics metrics.MetricOptions, name string, c *client.Client, logger *logrus.Logger) {
exists, err := lookupSwarmService(name, c)
if err != nil || exists == false {
if err != nil {
logger.Fatalln(err)
}
- w.WriteHeader(http.StatusInternalServerError)
+ writeHead(name, metrics, http.StatusInternalServerError, w)
w.Write([]byte("Error resolving service."))
defer r.Body.Close()
}
@@ -72,7 +81,6 @@ func lookupSwarmService(serviceName string, c *client.Client) (bool, error) {
}
func invokeService(w http.ResponseWriter, r *http.Request, metrics metrics.MetricOptions, service string, requestBody []byte, logger *logrus.Logger) {
- metrics.GatewayFunctionInvocation.WithLabelValues(service).Add(1)
stamp := strconv.FormatInt(time.Now().Unix(), 10)
@@ -89,7 +97,7 @@ func invokeService(w http.ResponseWriter, r *http.Request, metrics metrics.Metri
response, err := http.Post(url, r.Header.Get("Content-Type"), buf)
if err != nil {
logger.Infoln(err)
- w.WriteHeader(500)
+ writeHead(service, metrics, http.StatusInternalServerError, w)
buf := bytes.NewBufferString("Can't reach service: " + service)
w.Write(buf.Bytes())
return
@@ -98,7 +106,8 @@ func invokeService(w http.ResponseWriter, r *http.Request, metrics metrics.Metri
responseBody, readErr := ioutil.ReadAll(response.Body)
if readErr != nil {
fmt.Println(readErr)
- w.WriteHeader(500)
+
+ writeHead(service, metrics, http.StatusInternalServerError, w)
buf := bytes.NewBufferString("Error reading response from service: " + service)
w.Write(buf.Bytes())
return
@@ -107,7 +116,7 @@ func invokeService(w http.ResponseWriter, r *http.Request, metrics metrics.Metri
// Match header for strict services
w.Header().Set("Content-Type", r.Header.Get("Content-Type"))
- w.WriteHeader(http.StatusOK)
+ writeHead(service, metrics, http.StatusOK, w)
w.Write(responseBody)
seconds := time.Since(start).Seconds()
diff --git a/gateway/metrics/metrics.go b/gateway/metrics/metrics.go
index e28d8773..44bba64d 100644
--- a/gateway/metrics/metrics.go
+++ b/gateway/metrics/metrics.go
@@ -37,7 +37,7 @@ func BuildMetricsOptions() MetricOptions {
Name: "gateway_function_invocation_total",
Help: "Individual function metrics",
},
- []string{"function_name"},
+ []string{"function_name", "code"},
)
metricsOptions := MetricOptions{
--
GitLab