fixes #29 adding health status reporting options to slack and metrics…

… reporters, allowing the default report interval to be overriden per reporter
AppMetrics · Jul 13, 2018 · 747cfa2 · 747cfa2
1 parent 61314cc
commit 747cfa2
Show file tree

Hide file tree

Showing 13 changed files with 310 additions and 11 deletions.
diff --git a/sandbox/HealthSandbox/Host.cs b/sandbox/HealthSandbox/Host.cs
@@ -101,6 +101,7 @@ private static void Init()
                                          {
                                              options.Channel = SlackChannel;
                                              options.WebhookUrl = SlackWebhookUrl;
+                                             options.ReportInterval = TimeSpan.FromSeconds(30);
                                          })
                                      .Report.ToMetrics(Metrics)
                                      .HealthChecks.AddCheck(new SampleHealthCheck())

diff --git a/sandbox/HealthSandbox/SampleHealthStatusReporter.cs b/sandbox/HealthSandbox/SampleHealthStatusReporter.cs
@@ -11,6 +11,9 @@ namespace HealthSandbox
 {
     public class SampleHealthStatusReporter : IReportHealthStatus
     {
+        /// <inheritdoc />
+        public TimeSpan ReportInterval { get; set; }
+
         /// <inheritdoc />
         public Task ReportAsync(HealthOptions options, HealthStatus status, CancellationToken cancellationToken = default)
         {

diff --git a/src/App.Metrics.Health.Abstractions/Builder/IHealthReportingBuilder.cs b/src/App.Metrics.Health.Abstractions/Builder/IHealthReportingBuilder.cs
@@ -35,5 +35,18 @@ public interface IHealthReportingBuilder
         /// </returns>
         IHealthBuilder Using<TReportHealth>()
             where TReportHealth : IReportHealthStatus, new();
+
+        /// <summary>
+        ///     Reports metrics using the specified <see cref="IReportHealthStatus" />.
+        /// </summary>
+        /// <param name="reportInterval">The <see cref="TimeSpan" /> interval used to schedule health status reporting.</param>
+        /// <typeparam name="TReportHealth">
+        ///     An <see cref="IReportHealthStatus" /> type used to report health status.
+        /// </typeparam>
+        /// <returns>
+        ///     An <see cref="IHealthBuilder" /> that can be used to further configure App Metrics Health.
+        /// </returns>
+        IHealthBuilder Using<TReportHealth>(TimeSpan reportInterval)
+            where TReportHealth : IReportHealthStatus, new();
     }
 }
diff --git a/src/App.Metrics.Health.Abstractions/HealthConstants.cs b/src/App.Metrics.Health.Abstractions/HealthConstants.cs
@@ -2,6 +2,7 @@
 // Copyright (c) App Metrics Contributors. All rights reserved.
 // </copyright>
 
+using System;
 using System.Collections.Generic;
 using System.Collections.ObjectModel;
 
@@ -27,5 +28,10 @@ public static class HealthConstants
                     { HealthCheckStatus.Degraded, DegradedStatusDisplay },
                     { HealthCheckStatus.Ignored, IgnoredStatusDisplay }
                 });
+
+        public static class Reporting
+        {
+            public static readonly TimeSpan DefaultReportInterval = TimeSpan.FromSeconds(10);
+        }
     }
 }
diff --git a/src/App.Metrics.Health.Abstractions/IReportHealthStatus.cs b/src/App.Metrics.Health.Abstractions/IReportHealthStatus.cs
@@ -2,13 +2,20 @@
 // Copyright (c) App Metrics Contributors. All rights reserved.
 // </copyright>
 
+using System;
 using System.Threading;
 using System.Threading.Tasks;
 
 namespace App.Metrics.Health
 {
     public interface IReportHealthStatus
     {
+        /// <summary>
+        ///     Gets <see cref="TimeSpan" /> interval to flush metrics values. Defaults to
+        ///     <see cref="HealthConstants.Reporting.DefaultReportInterval" />.
+        /// </summary>
+        TimeSpan ReportInterval { get; set; }
+
         Task ReportAsync(HealthOptions options, HealthStatus status, CancellationToken cancellationToken = default);
     }
 }
diff --git a/src/App.Metrics.Health.Core/Builder/HealthReportingBuilder.cs b/src/App.Metrics.Health.Core/Builder/HealthReportingBuilder.cs
@@ -29,6 +29,8 @@ public IHealthBuilder Using(IReportHealthStatus reporter)
                 throw new ArgumentNullException(nameof(reporter));
             }
 
+            EnsureRequiredProperties(reporter);
+
             _reporters(reporter);
 
             return Builder;
@@ -42,5 +44,20 @@ public IHealthBuilder Using<TReportHealth>()
 
             return Using(reporter);
         }
+
+        public IHealthBuilder Using<TReportHealth>(TimeSpan reportInterval)
+            where TReportHealth : IReportHealthStatus, new()
+        {
+            var reporter = new TReportHealth { ReportInterval = reportInterval };
+
+            return Using(reporter);
+        }
+
+        private static void EnsureRequiredProperties(IReportHealthStatus reporter)
+        {
+            reporter.ReportInterval = reporter.ReportInterval <= TimeSpan.Zero
+                ? HealthConstants.Reporting.DefaultReportInterval
+                : reporter.ReportInterval;
+        }
     }
 }
diff --git a/src/App.Metrics.Health.Reporting.Metrics/Builder/HealthResultsAsMetricsBuilderExtensions.cs b/src/App.Metrics.Health.Reporting.Metrics/Builder/HealthResultsAsMetricsBuilderExtensions.cs
@@ -18,5 +18,15 @@ public static IHealthBuilder ToMetrics(
 
             return healthReportingBuilder.Builder;
         }
+
+        public static IHealthBuilder ToMetrics(
+            this IHealthReportingBuilder healthReportingBuilder,
+            IMetrics metrics,
+            HealthAsMetricsOptions options)
+        {
+            healthReportingBuilder.Using(new HealthResultsAsMetricsReporter(metrics, options));
+
+            return healthReportingBuilder.Builder;
+        }
     }
 }
diff --git a/src/App.Metrics.Health.Reporting.Metrics/HealthAsMetricsOptions.cs b/src/App.Metrics.Health.Reporting.Metrics/HealthAsMetricsOptions.cs
@@ -0,0 +1,24 @@
+// <copyright file="HealthAsMetricsOptions.cs" company="App Metrics Contributors">
+// Copyright (c) App Metrics Contributors. All rights reserved.
+// </copyright>
+
+using System;
+
+namespace App.Metrics.Health.Reporting.Metrics
+{
+    public class HealthAsMetricsOptions
+    {
+        /// <summary>
+        ///     Gets or sets the health status reporting interval.
+        /// </summary>
+        /// <remarks>
+        ///     If not set reporting interval will be set to the <see cref="HealthConstants.Reporting.DefaultReportInterval" />.
+        /// </remarks>
+        /// <value>
+        ///     The <see cref="TimeSpan" /> to wait between reporting health status.
+        /// </value>
+        public TimeSpan ReportInterval { get; set; }
+
+        public bool Enabled { get; set; } = true;
+    }
+}
diff --git a/src/App.Metrics.Health.Reporting.Metrics/HealthResultsAsMetricsReporter.cs b/src/App.Metrics.Health.Reporting.Metrics/HealthResultsAsMetricsReporter.cs
@@ -2,8 +2,10 @@
 // Copyright (c) App Metrics Contributors. All rights reserved.
 // </copyright>
 
+using System;
 using System.Threading;
 using System.Threading.Tasks;
+using App.Metrics.Health.Logging;
 #if !NETSTANDARD1_6
 using App.Metrics.Health.Internal;
  #endif
@@ -13,13 +15,46 @@ namespace App.Metrics.Health.Reporting.Metrics
 {
     public class HealthResultsAsMetricsReporter : IReportHealthStatus
     {
+        private static readonly ILog Logger = LogProvider.For<HealthResultsAsMetricsReporter>();
         private readonly IMetrics _metrics;
+        private readonly HealthAsMetricsOptions _healthAsMetricsOptions;
 
-        public HealthResultsAsMetricsReporter(IMetrics metrics) { _metrics = metrics; }
+        public HealthResultsAsMetricsReporter(IMetrics metrics)
+            : this(metrics, new HealthAsMetricsOptions())
+        {
+        }
+
+        public HealthResultsAsMetricsReporter(IMetrics metrics, HealthAsMetricsOptions healthAsMetricsOptions)
+        {
+            _healthAsMetricsOptions = healthAsMetricsOptions ?? throw new ArgumentNullException(nameof(healthAsMetricsOptions));
+            _metrics = metrics ?? throw new ArgumentNullException(nameof(metrics));
+
+            ReportInterval = healthAsMetricsOptions.ReportInterval > TimeSpan.Zero
+                ? healthAsMetricsOptions.ReportInterval
+                : HealthConstants.Reporting.DefaultReportInterval;
+
+            Logger.Trace($"Using Metrics Reporter {this}. ReportInterval: {ReportInterval}");
+        }
+
+        /// <inheritdoc />
+        public TimeSpan ReportInterval { get; set; }
 
         /// <inheritdoc />
         public Task ReportAsync(HealthOptions options, HealthStatus status, CancellationToken cancellationToken = default)
         {
+            if (!_healthAsMetricsOptions.Enabled || !options.Enabled)
+            {
+                Logger.Trace($"Health Status Reporter `{this}` disabled, not reporting.");
+
+#if NETSTANDARD1_6
+                return Task.CompletedTask;
+#else
+                return AppMetricsHealthTaskHelper.CompletedTask();
+#endif
+            }
+
+            Logger.Trace($"Health Status Reporter `{this}` reporting health status.");
+
             foreach (var healthResult in status.Results)
             {
                 var tags = new MetricTags(HealthReportingConstants.TagKeys.HealthCheckName, healthResult.Name);
@@ -51,6 +86,8 @@ public Task ReportAsync(HealthOptions options, HealthStatus status, Cancellation
 
             _metrics.Measure.Gauge.SetValue(ApplicationHealthMetricRegistry.HealthGauge, overallHealthStatus);
 
+            Logger.Trace($"Health Status Reporter `{this}` successfully reported health status.");
+
 #if NETSTANDARD1_6
             return Task.CompletedTask;
 #else

diff --git a/src/App.Metrics.Health.Reporting.Slack/SlackHealthAlertOptions.cs b/src/App.Metrics.Health.Reporting.Slack/SlackHealthAlertOptions.cs
@@ -2,20 +2,33 @@
 // Copyright (c) App Metrics Contributors. All rights reserved.
 // </copyright>
 
+using System;
+
 namespace App.Metrics.Health.Reporting.Slack
 {
     public class SlackHealthAlertOptions
     {
+        public bool AlertOnDegradedChecks { get; set; } = true;
+
         public string Channel { get; set; }
 
-        public string WebhookUrl { get; set; }
+        public string EmojiIcon { get; set; }
 
-        public string Username { get; set; }
+        public bool Enabled { get; set; } = true;
 
-        public string EmojiIcon { get; set; }
+        /// <summary>
+        ///     Gets or sets the health status reporting interval.
+        /// </summary>
+        /// <remarks>
+        ///     If not set reporting interval will be set to the <see cref="HealthConstants.Reporting.DefaultReportInterval" />.
+        /// </remarks>
+        /// <value>
+        ///     The <see cref="TimeSpan" /> to wait between reporting health status.
+        /// </value>
+        public TimeSpan ReportInterval { get; set; }
 
-        public bool AlertOnDegradedChecks { get; set; } = true;
+        public string Username { get; set; }
 
-        public bool Enabled { get; set; } = true;
+        public string WebhookUrl { get; set; }
     }
 }
diff --git a/src/App.Metrics.Health.Reporting.Slack/SlackIncomingWebHookHealthAlerter.cs b/src/App.Metrics.Health.Reporting.Slack/SlackIncomingWebHookHealthAlerter.cs
@@ -8,6 +8,7 @@
 using System.Net.Http;
 using System.Threading;
 using System.Threading.Tasks;
+using App.Metrics.Health.Logging;
 using App.Metrics.Health.Reporting.Slack.Internal;
 
 namespace App.Metrics.Health.Reporting.Slack
@@ -22,23 +23,38 @@ public class SlackIncomingWebHookHealthAlerter : IReportHealthStatus
         private static readonly DateTime Epoch = new DateTime(1970, 1, 1, 0, 0, 0, 0, DateTimeKind.Utc);
         private static readonly HashSet<string> LastUnhealthyCheckCache = new HashSet<string>();
         private static readonly HashSet<string> LastDegradedCheckCache = new HashSet<string>();
+        private static readonly ILog Logger = LogProvider.For<SlackIncomingWebHookHealthAlerter>();
         private readonly HttpClient _httpClient;
         private readonly SlackHealthAlertOptions _slackOptions;
 
         public SlackIncomingWebHookHealthAlerter(SlackHealthAlertOptions slackOptions)
         {
             _slackOptions = slackOptions ?? throw new ArgumentNullException(nameof(slackOptions));
+
+            ReportInterval = slackOptions.ReportInterval > TimeSpan.Zero
+                ? slackOptions.ReportInterval
+                : HealthConstants.Reporting.DefaultReportInterval;
+
             _httpClient = new HttpClient();
+
+            Logger.Trace($"Using Metrics Reporter {this}. WebhookUrl: {slackOptions.WebhookUrl} ReportInterval: {ReportInterval} Channel: {slackOptions.Channel}");
         }
 
+        /// <inheritdoc />
+        public TimeSpan ReportInterval { get; set; }
+
         /// <inheritdoc />
         public async Task ReportAsync(HealthOptions options, HealthStatus status, CancellationToken cancellationToken = default)
         {
-            if (!_slackOptions.Enabled)
+            if (!_slackOptions.Enabled || !options.Enabled)
             {
+                Logger.Trace($"Health Status Reporter `{this}` disabled, not reporting.");
+
                 return;
             }
 
+            Logger.Trace($"Health Status Reporter `{this}` reporting health status.");
+
             var applicationName = options.ApplicationName;
 
             if (Uri.TryCreate(applicationName, UriKind.Absolute, out var appUri))
@@ -73,10 +89,26 @@ public async Task ReportAsync(HealthOptions options, HealthStatus status, Cancel
 
             if (slackMessage.Attachments.Any())
             {
-                await _httpClient.PostAsync(_slackOptions.WebhookUrl, new JsonContent(slackMessage), cancellationToken);
-            }
+                try
+                {
+                    var response = await _httpClient.PostAsync(_slackOptions.WebhookUrl, new JsonContent(slackMessage), cancellationToken);
 
-            await AlertStatusChangeChecks(status, applicationName, cancellationToken);
+                    if (response.IsSuccessStatusCode)
+                    {
+                        Logger.Trace($"Health Status Reporter `{this}` successfully reported health status.");
+
+                        await AlertStatusChangeChecks(status, applicationName, cancellationToken);
+                    }
+                    else
+                    {
+                        Logger.Error($"Health Status Reporter `{this}` failed to reported health status with status code: `{response.StatusCode}` and reason phrase: `{response.ReasonPhrase}`");
+                    }
+                }
+                catch (Exception ex)
+                {
+                    Logger.Error(ex, $"Health Status Reporter `{this}` failed to reported health status");
+                }
+            }
         }
 
         private void AddHealthAttachments(
@@ -179,7 +211,25 @@ private async Task AlertStatusChangeChecks(HealthStatus status, string applicati
 
                 if (slackMessage.Attachments.Any())
                 {
-                    await _httpClient.PostAsync(_slackOptions.WebhookUrl, new JsonContent(slackMessage), cancellationToken);
+                    try
+                    {
+                        var response = await _httpClient.PostAsync(_slackOptions.WebhookUrl, new JsonContent(slackMessage), cancellationToken);
+
+                        if (response.IsSuccessStatusCode)
+                        {
+                            Logger.Trace($"Health Status Reporter `{this}` successfully reported health status changes.");
+
+                            await AlertStatusChangeChecks(status, applicationName, cancellationToken);
+                        }
+                        else
+                        {
+                            Logger.Error($"Health Status Reporter `{this}` failed to reported health status changes with status code: `{response.StatusCode}` and reason phrase: `{response.ReasonPhrase}`");
+                        }
+                    }
+                    catch (Exception ex)
+                    {
+                        Logger.Error(ex, $"Health Status Reporter `{this}` failed to reported health status changes.");
+                    }
                 }
             }
         }