Add Graceful Shutdown Support

packages/opentelemetry-core/src/platform/browser/ShutdownNotifier.ts

@@ -0,0 +1,32 @@
+/*
+ * Copyright The OpenTelemetry Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Adds an event listener to trigger a callback when an unload event in the window is detected
+ */
+export function notifyOnGlobalShutdown(cb: () => void): () => void {
+  window.addEventListener('unload', cb, { once: true });
+  return function removeCallbackFromGlobalShutdown() {
+    window.removeEventListener('unload', cb, false);
+  };
+}
+
+/**
+ * Warning: meant for internal use only! Closes the current window, triggering the unload event
+ */
+export function _invokeGlobalShutdown() {
+  window.close();
+}

packages/opentelemetry-core/src/platform/browser/index.ts

@@ -21,3 +21,4 @@ export * from './RandomIdGenerator';
 export * from './performance';
 export * from './sdk-info';
 export * from './timer-util';
+export * from './ShutdownNotifier';

packages/opentelemetry-core/src/platform/node/ShutdownNotifier.ts

@@ -0,0 +1,32 @@
+/*
+ * Copyright The OpenTelemetry Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      https://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Adds an event listener to trigger a callback when a SIGTERM is detected in the process
+ */
+export function notifyOnGlobalShutdown(cb: () => void): () => void {
+  process.once('SIGTERM', cb);
+  return function removeCallbackFromGlobalShutdown() {
+    process.removeListener('SIGTERM', cb);
+  };
+}
+
+/**
+ * Warning: meant for internal use only! Sends a SIGTERM to the current process
+ */
+export function _invokeGlobalShutdown() {
+  process.kill(process.pid, 'SIGTERM');
+}

packages/opentelemetry-core/src/platform/node/index.ts

@@ -21,3 +21,4 @@ export * from './RandomIdGenerator';
 export * from './performance';
 export * from './sdk-info';
 export * from './timer-util';
+export * from './ShutdownNotifier';

packages/opentelemetry-exporter-prometheus/test/prometheus.test.ts

@@ -15,6 +15,10 @@
  */
 
 import { HrTime, ObserverResult } from '@opentelemetry/api';
+import {
+  notifyOnGlobalShutdown,
+  _invokeGlobalShutdown,
+} from '@opentelemetry/core';
 import {
   CounterMetric,
   SumAggregator,
@@ -32,6 +36,7 @@ const mockedTimeMS = 1586347902211000;
 
 describe('PrometheusExporter', () => {
   let toPoint: () => Point<Sum>;
+  let removeEvent: Function | undefined;
   before(() => {
     toPoint = SumAggregator.prototype.toPoint;
     SumAggregator.prototype.toPoint = function (): Point<Sum> {
@@ -185,16 +190,27 @@ describe('PrometheusExporter', () => {
 
   describe('export', () => {
     let exporter: PrometheusExporter;
+    let meterProvider: MeterProvider;
     let meter: Meter;
 
     beforeEach(done => {
       exporter = new PrometheusExporter();
-      meter = new MeterProvider().getMeter('test-prometheus');
+      meterProvider = new MeterProvider({
+        interval: Math.pow(2, 31) - 1,
+        gracefulShutdown: true,
+      });
+      meter = meterProvider.getMeter('test-prometheus', '1', {
+        exporter: exporter,
+      });
       exporter.startServer(done);
     });
 
     afterEach(done => {
       exporter.shutdown(done);
+      if (removeEvent) {
+        removeEvent();
+        removeEvent = undefined;
+      }
     });
 
     it('should export a count aggregation', done => {
@@ -320,6 +336,70 @@ describe('PrometheusExporter', () => {
       });
     });
 
+    it('should export multiple labels on graceful shutdown', done => {
+      const counter = meter.createCounter('counter', {
+        description: 'a test description',
+      }) as CounterMetric;
+
+      counter.bind({ counterKey1: 'labelValue1' }).add(10);
+      counter.bind({ counterKey1: 'labelValue2' }).add(20);
+      counter.bind({ counterKey1: 'labelValue3' }).add(30);
+
+      removeEvent = notifyOnGlobalShutdown(() => {
+        http
+          .get('http://localhost:9464/metrics', res => {
+            res.on('data', chunk => {
+              const body = chunk.toString();
+              const lines = body.split('\n');
+
+              assert.deepStrictEqual(lines, [
+                '# HELP counter a test description',
+                '# TYPE counter counter',
+                `counter{counterKey1="labelValue1"} 10 ${mockedTimeMS}`,
+                `counter{counterKey1="labelValue2"} 20 ${mockedTimeMS}`,
+                `counter{counterKey1="labelValue3"} 30 ${mockedTimeMS}`,
+                '',
+              ]);
+
+              done();
+            });
+          })
+          .on('error', errorHandler(done));
+      });
+      _invokeGlobalShutdown();
+    });
+
+    it('should export multiple labels on manual shutdown', done => {
+      const counter = meter.createCounter('counter', {
+        description: 'a test description',
+      }) as CounterMetric;
+
+      counter.bind({ counterKey1: 'labelValue1' }).add(10);
+      counter.bind({ counterKey1: 'labelValue2' }).add(20);
+      counter.bind({ counterKey1: 'labelValue3' }).add(30);
+      meterProvider.shutdown(() => {
+        http
+          .get('http://localhost:9464/metrics', res => {
+            res.on('data', chunk => {
+              const body = chunk.toString();
+              const lines = body.split('\n');
+
+              assert.deepStrictEqual(lines, [
+                '# HELP counter a test description',
+                '# TYPE counter counter',
+                `counter{counterKey1="labelValue1"} 10 ${mockedTimeMS}`,
+                `counter{counterKey1="labelValue2"} 20 ${mockedTimeMS}`,
+                `counter{counterKey1="labelValue3"} 30 ${mockedTimeMS}`,
+                '',
+              ]);
+
+              done();
+            });
+          })
+          .on('error', errorHandler(done));
+      });
+    });
+
     it('should export a comment if no metrics are registered', done => {
       exporter.export([], () => {
         http

packages/opentelemetry-metrics/src/Meter.ts

@@ -40,6 +40,7 @@ export class Meter implements api.Meter {
   private readonly _batcher: Batcher;
   private readonly _resource: Resource;
   private readonly _instrumentationLibrary: InstrumentationLibrary;
+  private readonly _controller: PushController;
 
   /**
    * Constructs a new Meter instance.
@@ -55,7 +56,7 @@ export class Meter implements api.Meter {
     // start the push controller
     const exporter = config.exporter || new NoopExporter();
     const interval = config.interval;
-    new PushController(this, exporter, interval);
+    this._controller = new PushController(this, exporter, interval);
   }
 
   /**
@@ -309,6 +310,10 @@ export class Meter implements api.Meter {
     return this._batcher;
   }
 
+  async shutdown(): Promise<void> {
+    await this._controller.shutdown();
+  }
+
   /**
    * Registers metric to register.
    * @param name The name of the metric.

packages/opentelemetry-metrics/src/MeterProvider.ts

@@ -14,7 +14,7 @@
  * limitations under the License.
  */
 
-import { ConsoleLogger } from '@opentelemetry/core';
+import { ConsoleLogger, notifyOnGlobalShutdown } from '@opentelemetry/core';
 import * as api from '@opentelemetry/api';
 import { Resource } from '@opentelemetry/resources';
 import { Meter } from '.';
@@ -26,6 +26,7 @@ import { DEFAULT_CONFIG, MeterConfig } from './types';
 export class MeterProvider implements api.MeterProvider {
   private readonly _config: MeterConfig;
   private readonly _meters: Map<string, Meter> = new Map();
+  private _cleanNotifyOnGlobalShutdown: Function | undefined;
   readonly resource: Resource;
   readonly logger: api.Logger;
 
@@ -36,6 +37,11 @@ export class MeterProvider implements api.MeterProvider {
       logger: this.logger,
       resource: this.resource,
     });
+    if (this._config.gracefulShutdown) {
+      this._cleanNotifyOnGlobalShutdown = notifyOnGlobalShutdown(
+        this._shutdownAllMeters.bind(this)
+      );
+    }
   }
 
   /**
@@ -54,4 +60,23 @@ export class MeterProvider implements api.MeterProvider {
 
     return this._meters.get(key)!;
   }
+
+  shutdown(cb: () => void = () => {}): void {
+    this._shutdownAllMeters().then(() => {
+      setTimeout(cb, 0);
+    });
+    if (this._cleanNotifyOnGlobalShutdown) {
+      this._cleanNotifyOnGlobalShutdown();
+      this._cleanNotifyOnGlobalShutdown = undefined;
+    }
+  }
+
+  private _shutdownAllMeters() {
+    if (this._config.exporter) {
+      this._config.exporter.shutdown();
+    }
+    return Promise.all(
+      Array.from(this._meters, ([_, meter]) => meter.shutdown())
+    );
+  }
 }

packages/opentelemetry-metrics/src/export/Controller.ts

@@ -38,12 +38,24 @@ export class PushController extends Controller {
     unrefTimer(this._timer);
   }
 
-  private _collect() {
-    this._meter.collect();
-    this._exporter.export(this._meter.getBatcher().checkPointSet(), result => {
-      if (result !== ExportResult.SUCCESS) {
-        // @todo: log error
-      }
+  async shutdown(): Promise<void> {
+    await this._collect();
+  }
+
+  private async _collect(): Promise<void> {
+    await this._meter.collect();
+    return new Promise((resolve, reject) => {
+      this._exporter.export(
+        this._meter.getBatcher().checkPointSet(),
+        result => {
+          if (result === ExportResult.SUCCESS) {
+            resolve();
+          } else {
+            // @todo log error
+            reject();
+          }
+        }
+      );
     });
   }
 }

packages/opentelemetry-metrics/src/types.ts

@@ -39,11 +39,15 @@ export interface MeterConfig {
 
   /** Metric batcher. */
   batcher?: Batcher;
+
+  /** Bool for whether or not graceful shutdown is enabled. If disabled metrics will not be exported when SIGTERM is recieved */
+  gracefulShutdown?: boolean;
 }
 
 /** Default Meter configuration. */
 export const DEFAULT_CONFIG = {
   logLevel: getEnv().OTEL_LOG_LEVEL,
+  gracefulShutdown: true,
 };
 
 /** The default metric creation options value. */