From 7d66489cb4c6b67972e59e80350ca14a855d179d Mon Sep 17 00:00:00 2001 From: Brendan Burns Date: Fri, 9 Apr 2021 08:53:05 -0700 Subject: [PATCH] Add a Prometheus handler. (#591) * Add a Prometheus handler. * Address comments --- README.md | 10 ++ examples/prometheus/Prometheus.cs | 31 +++++ examples/prometheus/prometheus.csproj | 12 ++ src/KubernetesClient/KubernetesClient.csproj | 1 + .../KubernetesRequestDigest.cs | 112 ++++++++++++++++++ src/KubernetesClient/PrometheusHandler.cs | 58 +++++++++ 6 files changed, 224 insertions(+) create mode 100755 examples/prometheus/Prometheus.cs create mode 100755 examples/prometheus/prometheus.csproj create mode 100644 src/KubernetesClient/KubernetesRequestDigest.cs create mode 100644 src/KubernetesClient/PrometheusHandler.cs diff --git a/README.md b/README.md index c72dd18..df1d7b1 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,16 @@ methods are currently supported, but a few are not, see the You should also be able to authenticate using the in-cluster service account using the `InClusterConfig` function shown below. +## Monitoring +There is optional built-in metric generation for prometheus client metrics. +The metrics exported are: + +* `k8s_dotnet_request_total` - Counter of request, broken down by HTTP Method +* `k8s_dotnet_response_code_total` - Counter of responses, broken down by HTTP Method and response code +* `k8s_request_latency_seconds` - Latency histograms broken down by method, api group, api version and resource kind + +There is an example integrating these monitors in the examples/prometheus directory. + ## Sample Code ### Creating the client diff --git a/examples/prometheus/Prometheus.cs b/examples/prometheus/Prometheus.cs new file mode 100755 index 0000000..b15f361 --- /dev/null +++ b/examples/prometheus/Prometheus.cs @@ -0,0 +1,31 @@ +using System; +using System.Net.Http; +using System.Threading; +using k8s; +using k8s.Monitoring; +using Prometheus; + +namespace prom +{ + internal class Prometheus + { + private static void Main(string[] args) + { + var config = KubernetesClientConfiguration.BuildDefaultConfig(); + var handler = new PrometheusHandler(); + IKubernetes client = new Kubernetes(config, new DelegatingHandler[] { handler }); + + var server = new MetricServer(hostname: "localhost", port: 1234); + server.Start(); + + Console.WriteLine("Making requests!"); + while (true) + { + client.ListNamespacedPod("default"); + client.ListNode(); + client.ListNamespacedDeployment("default"); + Thread.Sleep(1000); + } + } + } +} diff --git a/examples/prometheus/prometheus.csproj b/examples/prometheus/prometheus.csproj new file mode 100755 index 0000000..028b76a --- /dev/null +++ b/examples/prometheus/prometheus.csproj @@ -0,0 +1,12 @@ + + + + + + + + Exe + net5 + + + diff --git a/src/KubernetesClient/KubernetesClient.csproj b/src/KubernetesClient/KubernetesClient.csproj index 3e6ea71..e835b5f 100644 --- a/src/KubernetesClient/KubernetesClient.csproj +++ b/src/KubernetesClient/KubernetesClient.csproj @@ -28,6 +28,7 @@ + diff --git a/src/KubernetesClient/KubernetesRequestDigest.cs b/src/KubernetesClient/KubernetesRequestDigest.cs new file mode 100644 index 0000000..cc7430c --- /dev/null +++ b/src/KubernetesClient/KubernetesRequestDigest.cs @@ -0,0 +1,112 @@ +// Derived from +// https://github.com/kubernetes-client/java/blob/master/util/src/main/java/io/kubernetes/client/apimachinery/KubernetesResource.java +using System; +using System.Net.Http; +using System.Text.RegularExpressions; +using System.Web; + +namespace k8s +{ + public class KubernetesRequestDigest + { + private static Regex resourcePattern = + new Regex(@"^/(api|apis)(/\S+)?/v\d\w*/\S+", RegexOptions.Compiled); + + public string Path { get; } + public bool IsNonResourceRequest { get; } + public string ApiGroup { get; } + public string ApiVersion { get; } + public string Kind { get; } + public string Verb { get; } + + public KubernetesRequestDigest(string urlPath, bool isNonResourceRequest, string apiGroup, string apiVersion, string kind, string verb) + { + this.Path = urlPath; + this.IsNonResourceRequest = isNonResourceRequest; + this.ApiGroup = apiGroup; + this.ApiVersion = apiVersion; + this.Kind = kind; + this.Verb = verb; + } + + public static KubernetesRequestDigest Parse(HttpRequestMessage request) + { + if (request == null) + { + throw new ArgumentNullException(nameof(request)); + } + + string urlPath = request.RequestUri.AbsolutePath; + if (!IsResourceRequest(urlPath)) + { + return NonResource(urlPath); + } + + try + { + string apiGroup; + string apiVersion; + string kind; + + var parts = urlPath.Split('/'); + var namespaced = urlPath.IndexOf("/namespaces/", StringComparison.Ordinal) != -1; + + if (urlPath.StartsWith("/api/v1", StringComparison.Ordinal)) + { + apiGroup = ""; + apiVersion = "v1"; + + if (namespaced) + { + kind = parts[5]; + } + else + { + kind = parts[3]; + } + } + else + { + apiGroup = parts[2]; + apiVersion = parts[3]; + if (namespaced) + { + kind = parts[6]; + } + else + { + kind = parts[4]; + } + } + + return new KubernetesRequestDigest( + urlPath, + false, + apiGroup, + apiVersion, + kind, + HasWatchParameter(request) ? "WATCH" : request.Method.ToString()); + } + catch (Exception) + { + return NonResource(urlPath); + } + } + + private static KubernetesRequestDigest NonResource(string urlPath) + { + KubernetesRequestDigest digest = new KubernetesRequestDigest(urlPath, true, "nonresource", "na", "na", "na"); + return digest; + } + + public static bool IsResourceRequest(string urlPath) + { + return resourcePattern.Matches(urlPath).Count > 0; + } + + private static bool HasWatchParameter(HttpRequestMessage request) + { + return !string.IsNullOrEmpty(HttpUtility.ParseQueryString(request.RequestUri.Query).Get("watch")); + } + } +} diff --git a/src/KubernetesClient/PrometheusHandler.cs b/src/KubernetesClient/PrometheusHandler.cs new file mode 100644 index 0000000..1447ede --- /dev/null +++ b/src/KubernetesClient/PrometheusHandler.cs @@ -0,0 +1,58 @@ +using Prometheus; +using System; +using System.Net.Http; +using System.Threading; +using System.Threading.Tasks; + +namespace k8s.Monitoring +{ + public class PrometheusHandler : DelegatingHandler + { + private const string PREFIX = "k8s_dotnet"; + private readonly Counter requests = Metrics.CreateCounter( + $"{PREFIX}_request_total", "Number of requests sent by this client", + new CounterConfiguration + { + LabelNames = new[] { "method" }, + }); + + private readonly Histogram requestLatency = Metrics.CreateHistogram( + $"{PREFIX}_request_latency_seconds", "Latency of requests sent by this client", + new HistogramConfiguration + { + LabelNames = new[] { "verb", "group", "version", "kind" }, + }); + + private readonly Counter responseCodes = Metrics.CreateCounter( + $"{PREFIX}_response_code_total", "Number of response codes received by the client", + new CounterConfiguration + { + LabelNames = new[] { "method", "code" }, + }); + + private readonly Gauge activeRequests = Metrics.CreateGauge( + $"{PREFIX}_active_requests", "Number of requests currently in progress", + new GaugeConfiguration + { + LabelNames = new[] { "method" }, + }); + + protected override async Task SendAsync(HttpRequestMessage request, CancellationToken cancellationToken) + { + if (request == null) + { + throw new ArgumentNullException(nameof(request)); + } + + var digest = KubernetesRequestDigest.Parse(request); + requests.WithLabels(digest.Verb).Inc(); + using (activeRequests.WithLabels(digest.Verb).TrackInProgress()) + using (requestLatency.WithLabels(digest.Verb, digest.ApiGroup, digest.ApiVersion, digest.Kind).NewTimer()) + { + var resp = await base.SendAsync(request, cancellationToken).ConfigureAwait(false); + responseCodes.WithLabels(request.Method.ToString(), ((int)resp.StatusCode).ToString()).Inc(); + return resp; + } + } + } +}