Add a Prometheus handler. (#591)

* Add a Prometheus handler.

* Address comments

README.md

@@ -19,6 +19,16 @@ methods are currently supported, but a few are not, see the
 You should also be able to authenticate using the in-cluster service
 account using the `InClusterConfig` function shown below.
 
+## Monitoring
+There is optional built-in metric generation for prometheus client metrics.
+The metrics exported are:
+
+* `k8s_dotnet_request_total` - Counter of request, broken down by HTTP Method
+* `k8s_dotnet_response_code_total` - Counter of responses, broken down by HTTP Method and response code
+* `k8s_request_latency_seconds` - Latency histograms broken down by method, api group, api version and resource kind
+
+There is an example integrating these monitors in the examples/prometheus directory.
+
 ## Sample Code
 
 ### Creating the client

examples/prometheus/Prometheus.cs

@@ -0,0 +1,31 @@
+using System;
+using System.Net.Http;
+using System.Threading;
+using k8s;
+using k8s.Monitoring;
+using Prometheus;
+
+namespace prom
+{
+    internal class Prometheus
+    {
+        private static void Main(string[] args)
+        {
+            var config = KubernetesClientConfiguration.BuildDefaultConfig();
+            var handler = new PrometheusHandler();
+            IKubernetes client = new Kubernetes(config, new DelegatingHandler[] { handler });
+
+            var server = new MetricServer(hostname: "localhost", port: 1234);
+            server.Start();
+
+            Console.WriteLine("Making requests!");
+            while (true)
+            {
+                client.ListNamespacedPod("default");
+                client.ListNode();
+                client.ListNamespacedDeployment("default");
+                Thread.Sleep(1000);
+            }
+        }
+    }
+}

examples/prometheus/prometheus.csproj

@@ -0,0 +1,12 @@
+<Project Sdk="Microsoft.NET.Sdk">
+
+  <ItemGroup>
+    <ProjectReference Include="..\..\src\KubernetesClient\KubernetesClient.csproj" />
+  </ItemGroup>
+
+  <PropertyGroup>
+    <OutputType>Exe</OutputType>
+    <TargetFramework>net5</TargetFramework>
+  </PropertyGroup>
+
+</Project>

src/KubernetesClient/KubernetesClient.csproj

@@ -28,6 +28,7 @@
     <PackageReference Include="Nerdbank.GitVersioning" Version="3.3.37" PrivateAssets="all" />
     <PackageReference Include="Portable.BouncyCastle" Version="1.8.1.3" />
     <PackageReference Include="Microsoft.Rest.ClientRuntime" Version="2.3.10" />
+    <PackageReference Include="prometheus-net" Version="4.1.1" />
     <PackageReference Include="YamlDotNet" Version="8.1.2" />
     <PackageReference Include="Microsoft.SourceLink.GitHub" Version="1.0.0" PrivateAssets="All" />
     <PackageReference Include="System.Buffers" Version="4.5.1" />

src/KubernetesClient/KubernetesRequestDigest.cs

@@ -0,0 +1,112 @@
+// Derived from
+// https://github.com/kubernetes-client/java/blob/master/util/src/main/java/io/kubernetes/client/apimachinery/KubernetesResource.java
+using System;
+using System.Net.Http;
+using System.Text.RegularExpressions;
+using System.Web;
+
+namespace k8s
+{
+    public class KubernetesRequestDigest
+    {
+        private static Regex resourcePattern =
+            new Regex(@"^/(api|apis)(/\S+)?/v\d\w*/\S+", RegexOptions.Compiled);
+
+        public string Path { get; }
+        public bool IsNonResourceRequest { get; }
+        public string ApiGroup { get; }
+        public string ApiVersion { get; }
+        public string Kind { get; }
+        public string Verb { get; }
+
+        public KubernetesRequestDigest(string urlPath, bool isNonResourceRequest, string apiGroup, string apiVersion, string kind, string verb)
+        {
+            this.Path = urlPath;
+            this.IsNonResourceRequest = isNonResourceRequest;
+            this.ApiGroup = apiGroup;
+            this.ApiVersion = apiVersion;
+            this.Kind = kind;
+            this.Verb = verb;
+        }
+
+        public static KubernetesRequestDigest Parse(HttpRequestMessage request)
+        {
+            if (request == null)
+            {
+                throw new ArgumentNullException(nameof(request));
+            }
+
+            string urlPath = request.RequestUri.AbsolutePath;
+            if (!IsResourceRequest(urlPath))
+            {
+                return NonResource(urlPath);
+            }
+
+            try
+            {
+                string apiGroup;
+                string apiVersion;
+                string kind;
+
+                var parts = urlPath.Split('/');
+                var namespaced = urlPath.IndexOf("/namespaces/", StringComparison.Ordinal) != -1;
+
+                if (urlPath.StartsWith("/api/v1", StringComparison.Ordinal))
+                {
+                    apiGroup = "";
+                    apiVersion = "v1";
+
+                    if (namespaced)
+                    {
+                        kind = parts[5];
+                    }
+                    else
+                    {
+                        kind = parts[3];
+                    }
+                }
+                else
+                {
+                    apiGroup = parts[2];
+                    apiVersion = parts[3];
+                    if (namespaced)
+                    {
+                        kind = parts[6];
+                    }
+                    else
+                    {
+                        kind = parts[4];
+                    }
+                }
+
+                return new KubernetesRequestDigest(
+                    urlPath,
+                    false,
+                    apiGroup,
+                    apiVersion,
+                    kind,
+                    HasWatchParameter(request) ? "WATCH" : request.Method.ToString());
+            }
+            catch (Exception)
+            {
+                return NonResource(urlPath);
+            }
+        }
+
+        private static KubernetesRequestDigest NonResource(string urlPath)
+        {
+            KubernetesRequestDigest digest = new KubernetesRequestDigest(urlPath, true, "nonresource", "na", "na", "na");
+            return digest;
+        }
+
+        public static bool IsResourceRequest(string urlPath)
+        {
+            return resourcePattern.Matches(urlPath).Count > 0;
+        }
+
+        private static bool HasWatchParameter(HttpRequestMessage request)
+        {
+            return !string.IsNullOrEmpty(HttpUtility.ParseQueryString(request.RequestUri.Query).Get("watch"));
+        }
+    }
+}

src/KubernetesClient/PrometheusHandler.cs

@@ -0,0 +1,58 @@
+using Prometheus;
+using System;
+using System.Net.Http;
+using System.Threading;
+using System.Threading.Tasks;
+
+namespace k8s.Monitoring
+{
+    public class PrometheusHandler : DelegatingHandler
+    {
+        private const string PREFIX = "k8s_dotnet";
+        private readonly Counter requests = Metrics.CreateCounter(
+            $"{PREFIX}_request_total", "Number of requests sent by this client",
+            new CounterConfiguration
+            {
+                LabelNames = new[] { "method" },
+            });
+
+        private readonly Histogram requestLatency = Metrics.CreateHistogram(
+            $"{PREFIX}_request_latency_seconds", "Latency of requests sent by this client",
+            new HistogramConfiguration
+            {
+                LabelNames = new[] { "verb", "group", "version", "kind" },
+            });
+
+        private readonly Counter responseCodes = Metrics.CreateCounter(
+            $"{PREFIX}_response_code_total", "Number of response codes received by the client",
+            new CounterConfiguration
+            {
+                LabelNames = new[] { "method", "code" },
+            });
+
+        private readonly Gauge activeRequests = Metrics.CreateGauge(
+            $"{PREFIX}_active_requests", "Number of requests currently in progress",
+            new GaugeConfiguration
+            {
+                LabelNames = new[] { "method" },
+            });
+
+        protected override async Task<HttpResponseMessage> SendAsync(HttpRequestMessage request, CancellationToken cancellationToken)
+        {
+            if (request == null)
+            {
+                throw new ArgumentNullException(nameof(request));
+            }
+
+            var digest = KubernetesRequestDigest.Parse(request);
+            requests.WithLabels(digest.Verb).Inc();
+            using (activeRequests.WithLabels(digest.Verb).TrackInProgress())
+            using (requestLatency.WithLabels(digest.Verb, digest.ApiGroup, digest.ApiVersion, digest.Kind).NewTimer())
+            {
+                var resp = await base.SendAsync(request, cancellationToken).ConfigureAwait(false);
+                responseCodes.WithLabels(request.Method.ToString(), ((int)resp.StatusCode).ToString()).Inc();
+                return resp;
+            }
+        }
+    }
+}