Skip to content

Commit

Permalink
Updating how cross-application correlations are tracked (#231)
Browse files Browse the repository at this point in the history
* Updating how cross-application correlations are tracked

Instead of using a hash of the instrumentation key we now use the appId, matching the .NET sdk.
We also use different headers to match the .NET sdk.

* Updating to only issue appId requests once per ikey

* Exposing profileQueryEndpoint property

Allows for the appId query target to be configured separately to the telemetry endpoint.
It may be specified either by the APPINSIGHTS_PROFILE_QUERY_ENDPOINT environment variable,
or explicitly via client.config.profileQueryEndpoint. Note that it must be set synchronously
with the creation of the Config otherwise the value will not be used.

* Allowing appId lookups to be cancelled if a new endpoint is specified

* Adding operationId to outbound HTTP headers
  • Loading branch information
MSLaguana authored and OsvaldoRosado committed May 5, 2017
1 parent 8c31b39 commit 6f8d4a2
Show file tree
Hide file tree
Showing 10 changed files with 225 additions and 85 deletions.
14 changes: 8 additions & 6 deletions AutoCollection/ClientRequestParser.ts
Original file line number Diff line number Diff line change
Expand Up @@ -8,12 +8,13 @@ import Logging = require("../Library/Logging");
import Util = require("../Library/Util");
import RequestResponseHeaders = require("../Library/RequestResponseHeaders");
import RequestParser = require("./RequestParser");
import CorrelationIdManager = require("../Library/CorrelationIdManager");

/**
* Helper class to read data from the requst/response objects and convert them into the telemetry contract
*/
class ClientRequestParser extends RequestParser {
private targetIKeyHash: string;
private correlationId: string;

constructor(requestOptions: string | http.RequestOptions | https.RequestOptions, request: http.ClientRequest) {
super();
Expand All @@ -38,8 +39,7 @@ class ClientRequestParser extends RequestParser {
*/
public onResponse(response: http.ClientResponse, properties?: { [key: string]: string }) {
this._setStatus(response.statusCode, undefined, properties);
this.targetIKeyHash =
response.headers && response.headers[RequestResponseHeaders.targetInstrumentationKeyHeader];
this.correlationId = Util.getCorrelationContextTarget(response, RequestResponseHeaders.requestContextTargetKey);
}

/**
Expand All @@ -54,12 +54,14 @@ class ClientRequestParser extends RequestParser {
let remoteDependency = new Contracts.RemoteDependencyData();
remoteDependency.type = Contracts.RemoteDependencyDataConstants.TYPE_HTTP;

if (this.targetIKeyHash) {
remoteDependency.target = urlObject.hostname;
if (this.correlationId) {
remoteDependency.type = Contracts.RemoteDependencyDataConstants.TYPE_AI;
remoteDependency.target = urlObject.hostname + " | " + this.targetIKeyHash;
if (this.correlationId !== CorrelationIdManager.correlationIdPrefix) {
remoteDependency.target = urlObject.hostname + " | " + this.correlationId;
}
} else {
remoteDependency.type = Contracts.RemoteDependencyDataConstants.TYPE_HTTP;
remoteDependency.target = urlObject.hostname;
}

remoteDependency.name = dependencyName;
Expand Down
29 changes: 22 additions & 7 deletions AutoCollection/ClientRequests.ts
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ import Logging = require("../Library/Logging");
import Util = require("../Library/Util");
import RequestResponseHeaders = require("../Library/RequestResponseHeaders");
import ClientRequestParser = require("./ClientRequestParser");
import { CorrelationContextManager, CorrelationContext } from "./CorrelationContextManager";

class AutoCollectClientRequests {
public static disableCollectionRequestOption = 'disableAppInsightsAutoCollection';
Expand Down Expand Up @@ -79,16 +80,30 @@ class AutoCollectClientRequests {

let requestParser = new ClientRequestParser(requestOptions, request);

// Add the source ikey hash to the request headers, if a value was not already provided.
// Add the source correlationId to the request headers, if a value was not already provided.
// The getHeader/setHeader methods aren't available on very old Node versions, and
// are not included in the v0.10 type declarations currently used. So check if the
// methods exist before invoking them.
if (client.config && client.config.instrumentationKeyHash &&
Util.canIncludeCorrelationHeader(client, requestParser.getUrl()) &&
request['getHeader'] && request['setHeader'] &&
!request['getHeader'](RequestResponseHeaders.sourceInstrumentationKeyHeader)) {
request['setHeader'](RequestResponseHeaders.sourceInstrumentationKeyHeader,
client.config.instrumentationKeyHash);
if (Util.canIncludeCorrelationHeader(client, requestParser.getUrl()) &&
request['getHeader'] && request['setHeader']) {
if (client.config && client.config.correlationId) {
const correlationHeader = request['getHeader'](RequestResponseHeaders.requestContextHeader);
if (correlationHeader) {
const components = correlationHeader.split(",");
const key = `${RequestResponseHeaders.requestContextSourceKey}=`;
if (!components.some((value) => value.substring(0,key.length) === key)) {
request['setHeader'](RequestResponseHeaders.requestContextHeader, `${correlationHeader},${RequestResponseHeaders.requestContextSourceKey}=${client.config.correlationId}`);
}
} else {
request['setHeader'](RequestResponseHeaders.requestContextHeader, `${RequestResponseHeaders.requestContextSourceKey}=${client.config.correlationId}`);
}
}

const currentContext = CorrelationContextManager.getCurrentContext();
if (currentContext && currentContext.operation) {
request['setHeader'](RequestResponseHeaders.parentIdHeader, currentContext.operation.id);
request['setHeader'](RequestResponseHeaders.rootIdHeader, currentContext.operation.parentId);
}
}

// Collect dependency telemetry about the request when it finishes.
Expand Down
7 changes: 3 additions & 4 deletions AutoCollection/ServerRequestParser.ts
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,7 @@ class ServerRequestParser extends RequestParser {
private connectionRemoteAddress:string;
private legacySocketRemoteAddress:string;
private userAgent: string;
private sourceIKeyHash: string;
private sourceCorrelationId: string;
private parentId: string;
private operationId: string;
private requestId: string;
Expand All @@ -34,8 +34,7 @@ class ServerRequestParser extends RequestParser {
this.rawHeaders = request.headers || (<any>request).rawHeaders;
this.socketRemoteAddress = (<any>request).socket && (<any>request).socket.remoteAddress;
this.userAgent = request.headers && request.headers["user-agent"];
this.sourceIKeyHash =
request.headers && request.headers[RequestResponseHeaders.sourceInstrumentationKeyHeader];
this.sourceCorrelationId = Util.getCorrelationContextTarget(request, RequestResponseHeaders.requestContextSourceKey);
this.parentId =
request.headers && request.headers[RequestResponseHeaders.parentIdHeader];
this.operationId =
Expand Down Expand Up @@ -64,7 +63,7 @@ class ServerRequestParser extends RequestParser {
requestData.id = this.requestId;
requestData.name = this.method + " " + url.parse(this.url).pathname;
requestData.url = this.url;
requestData.source = this.sourceIKeyHash;
requestData.source = this.sourceCorrelationId;
requestData.duration = Util.msToTimeSpan(this.duration);
requestData.responseCode = this.statusCode ? this.statusCode.toString() : null;
requestData.success = this._isSuccess();
Expand Down
26 changes: 16 additions & 10 deletions AutoCollection/ServerRequests.ts
Original file line number Diff line number Diff line change
Expand Up @@ -121,7 +121,7 @@ class AutoCollectServerRequests {
return;
}

AutoCollectServerRequests.addResponseIKeyHeader(client, response);
AutoCollectServerRequests.addResponseCorrelationIdHeader(client, response);

// store data about the request
var correlationContext = CorrelationContextManager.getCurrentContext();
Expand Down Expand Up @@ -151,7 +151,7 @@ class AutoCollectServerRequests {
var requestParser = _requestParser || new ServerRequestParser(request, correlationContext && correlationContext.operation.parentId || Util.newGuid());

if (Util.canIncludeCorrelationHeader(client, requestParser.getUrl())) {
AutoCollectServerRequests.addResponseIKeyHeader(client, response);
AutoCollectServerRequests.addResponseCorrelationIdHeader(client, response);
}

// Overwrite correlation context with request parser results (if not an automatic track. we've already precalculated the correlation context in that case)
Expand All @@ -177,15 +177,21 @@ class AutoCollectServerRequests {
}

/**
* Add the target ikey hash to the response headers, if not already provided.
* Add the target correlationId to the response headers, if not already provided.
*/
private static addResponseIKeyHeader(client:Client, response:http.ServerResponse) {
if (client.config && client.config.instrumentationKeyHash &&
response.getHeader && response.setHeader &&
!response.getHeader(RequestResponseHeaders.targetInstrumentationKeyHeader) &&
!(<any>response).headersSent) {
response.setHeader(RequestResponseHeaders.targetInstrumentationKeyHeader,
client.config.instrumentationKeyHash);
private static addResponseCorrelationIdHeader(client:Client, response:http.ServerResponse) {
if (client.config && client.config.correlationId &&
response.getHeader && response.setHeader && !(<any>response).headersSent) {
const correlationHeader = response.getHeader(RequestResponseHeaders.requestContextHeader);
if (correlationHeader) {
const components = correlationHeader.split(",");
const key = `${RequestResponseHeaders.requestContextSourceKey}=`;
if (!components.some((value) => value.substring(0,key.length) === key)) {
response.setHeader(RequestResponseHeaders.requestContextHeader, `${correlationHeader},${RequestResponseHeaders.requestContextSourceKey}=${client.config.correlationId}`);
}
} else {
response.setHeader(RequestResponseHeaders.requestContextHeader, `${RequestResponseHeaders.requestContextSourceKey}=${client.config.correlationId}`);
}
}
}

Expand Down
41 changes: 30 additions & 11 deletions Library/Config.ts
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
import crypto = require('crypto');
import CorrelationIdManager = require('./CorrelationIdManager');

class Config {

Expand All @@ -8,37 +8,63 @@ class Config {
// This key is provided in the readme
public static ENV_iKey = "APPINSIGHTS_INSTRUMENTATIONKEY";
public static legacy_ENV_iKey = "APPINSIGHTS_INSTRUMENTATION_KEY";
public static ENV_profileQueryEndpoint = "APPINSIGHTS_PROFILE_QUERY_ENDPOINT";

public instrumentationKey: string;
public instrumentationKeyHash: string;
public correlationId: string;
public sessionRenewalMs: number;
public sessionExpirationMs: number;
public endpointUrl: string;
public maxBatchSize: number;
public maxBatchIntervalMs: number;
public disableAppInsights: boolean;
public samplingPercentage: number;
public correlationIdRetryIntervalMs: number;

private endpointBase: string = "https://dc.services.visualstudio.com";
private setCorrelationId: (string) => void;
private _profileQueryEndpoint: string;

// A list of domains for which correlation headers will not be added.
public correlationHeaderExcludedDomains: string[];

constructor(instrumentationKey?: string) {
this.instrumentationKey = instrumentationKey || Config._getInstrumentationKey();
this.instrumentationKeyHash = Config._getStringHashBase64(this.instrumentationKey);
this.endpointUrl = "https://dc.services.visualstudio.com/v2/track";
this.endpointUrl = `${this.endpointBase}/v2/track`;
this.sessionRenewalMs = 30 * 60 * 1000;
this.sessionExpirationMs = 24 * 60 * 60 * 1000;
this.maxBatchSize = 250;
this.maxBatchIntervalMs = 15000;
this.disableAppInsights = false;
this.samplingPercentage = 100;
this.correlationIdRetryIntervalMs = 30 * 1000;
this.correlationHeaderExcludedDomains = [
"*.blob.core.windows.net",
"*.blob.core.chinacloudapi.cn",
"*.blob.core.cloudapi.de",
"*.blob.core.usgovcloudapi.net"];

this.setCorrelationId = (correlationId) => this.correlationId = correlationId;

this.profileQueryEndpoint = process.env[Config.ENV_profileQueryEndpoint] || this.endpointBase;
}

public set profileQueryEndpoint(endpoint: string) {
CorrelationIdManager.cancelCorrelationIdQuery(this._profileQueryEndpoint, this.instrumentationKey, this.setCorrelationId);
this._profileQueryEndpoint = endpoint;
this.correlationId = CorrelationIdManager.correlationIdPrefix; // Reset the correlationId while we wait for the new query
CorrelationIdManager.queryCorrelationId(
this._profileQueryEndpoint,
this.instrumentationKey,
this.correlationIdRetryIntervalMs,
this.setCorrelationId);
}

public get profileQueryEndpoint() {
return this._profileQueryEndpoint;
}


private static _getInstrumentationKey(): string {
// check for both the documented env variable and the azure-prefixed variable
var iKey = process.env[Config.ENV_iKey]
Expand All @@ -51,13 +77,6 @@ class Config {

return iKey;
}

private static _getStringHashBase64(value: string): string {
let hash = crypto.createHash('sha256');
hash.update(value);
let result = hash.digest('base64');
return result;
}
}

export = Config;
95 changes: 95 additions & 0 deletions Library/CorrelationIdManager.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,95 @@
import https = require('https');
import http = require('http');
import url = require('url');

class CorrelationIdManager {
public static correlationIdPrefix: "cid-v1:";

// To avoid extraneous HTTP requests, we maintain a queue of callbacks waiting on a particular appId lookup,
// as well as a cache of completed lookups so future requests can be resolved immediately.
private static pendingLookups: {[key: string]: Function[]} = {};
private static completedLookups: {[key: string]: string} = {};

public static queryCorrelationId(endpointBase: string, instrumentationKey: string, correlationIdRetryInterval: number, callback: (correlationId: string) => void) {
// GET request to `${this.endpointBase}/api/profiles/${this.instrumentationKey}/appId`
// If it 404s, the iKey is bad and we should give up
// If it fails otherwise, try again later
const appIdUrlString = `${endpointBase}/api/profiles/${instrumentationKey}/appId`;
const appIdUrl = url.parse(appIdUrlString);

if (CorrelationIdManager.completedLookups.hasOwnProperty(appIdUrlString)) {
callback(CorrelationIdManager.completedLookups[appIdUrlString]);
return;
} else if (CorrelationIdManager.pendingLookups[appIdUrlString]) {
CorrelationIdManager.pendingLookups[appIdUrlString].push(callback);
return;
}

CorrelationIdManager.pendingLookups[appIdUrlString] = [callback];

const requestOptions = {
protocol: appIdUrl.protocol,
hostname: appIdUrl.host,
path: appIdUrl.pathname,
method: 'GET',
// Ensure this request is not captured by auto-collection.
// Note: we don't refer to the property in ClientRequestParser because that would cause a cyclical dependency
disableAppInsightsAutoCollection: true
};

let httpRequest = appIdUrl.protocol === 'https:' ? https.request : http.request;

const fetchAppId = () => {
if (!CorrelationIdManager.pendingLookups[appIdUrlString]) {
// This query has been cancelled.
return;
}
const req = httpRequest(requestOptions, (res) => {
if (res.statusCode === 200) {
// Success; extract the appId from the body
let appId = "";
res.setEncoding("utf-8");
res.on('data', function (data) {
appId += data;
});
res.on('end', () => {
const result = CorrelationIdManager.correlationIdPrefix + appId;
CorrelationIdManager.completedLookups[appIdUrlString] = result;
if (CorrelationIdManager.pendingLookups[appIdUrlString]) {
CorrelationIdManager.pendingLookups[appIdUrlString].forEach((cb) => cb(result));
}
delete CorrelationIdManager.pendingLookups[appIdUrlString];
});
} else if (res.statusCode >= 400 && res.statusCode < 500) {
// Not found, probably a bad key. Do not try again.
CorrelationIdManager.completedLookups[appIdUrlString] = undefined;
delete CorrelationIdManager.pendingLookups[appIdUrlString];
} else {
// Retry after timeout.
setTimeout(fetchAppId, correlationIdRetryInterval);
}
});
if (req) {
req.on('error', () => {
// Unable to contact endpoint.
// Do nothing for now.
});
req.end();
}
};
setTimeout(fetchAppId, 0);
}

public static cancelCorrelationIdQuery(endpointBase: string, instrumentationKey: string, callback: (correlationId: string) => void) {
const appIdUrlString = `${endpointBase}/api/profiles/${instrumentationKey}/appId`;
const pendingLookups = CorrelationIdManager.pendingLookups[appIdUrlString];
if (pendingLookups) {
CorrelationIdManager.pendingLookups[appIdUrlString] = pendingLookups.filter((cb) => cb != callback);
if (CorrelationIdManager.pendingLookups[appIdUrlString].length == 0) {
delete CorrelationIdManager.pendingLookups[appIdUrlString];
}
}
}
}

export = CorrelationIdManager;
6 changes: 4 additions & 2 deletions Library/RequestResponseHeaders.ts
Original file line number Diff line number Diff line change
@@ -1,15 +1,17 @@
export = {

requestContextHeader: "Request-Context",
/**
* Source instrumentation header that is added by an application while making http
* requests and retrieved by the other application when processing incoming requests.
*/
sourceInstrumentationKeyHeader: "x-ms-request-source-ikey",
requestContextSourceKey: "appId",

/**
* Target instrumentation header that is added to the response and retrieved by the
* calling application when processing incoming responses.
*/
targetInstrumentationKeyHeader: "x-ms-request-target-ikey",
requestContextTargetKey: "appId",

/**
* Header containing the id of the immidiate caller
Expand Down
Loading

0 comments on commit 6f8d4a2

Please sign in to comment.