From 3643b202eefb4900a3b776cc6493fe8ab50b9502 Mon Sep 17 00:00:00 2001 From: Mayur Kale Date: Fri, 10 Jan 2020 15:36:53 -0800 Subject: [PATCH 1/5] add benchmark README and latest numbers --- benchmark/README.md | 51 +++++++++++++++++++++++++++++++++++++++++++++ benchmark/tracer.js | 3 ++- 2 files changed, 53 insertions(+), 1 deletion(-) create mode 100644 benchmark/README.md diff --git a/benchmark/README.md b/benchmark/README.md new file mode 100644 index 00000000000..d1c37b781bf --- /dev/null +++ b/benchmark/README.md @@ -0,0 +1,51 @@ +# Benchmarks + +## How to run + +To run your benchmark, just: +``` +$ npm run bench +``` + +If you're interested in writing benchmarking for other APIs, please write a benchmark in the `benchmark/index.js` module. Please refer to the `benchmark/tracer.js` or `benchmark/propagator.js` for more comprehensive examples. + +## Results + +# `0.3.3` release + +``` +Beginning BasicTracerRegistry Benchmark... + 7 tests completed. + + #startSpan x 79,704 ops/sec ±4.46% (10 runs sampled) + #startSpan:parent x 55,975 ops/sec ±1.90% (10 runs sampled) + #startSpan with attribute x 84,479 ops/sec ±2.82% (10 runs sampled) + #startSpan with 30 attributes x 36,239 ops/sec ±2.67% (10 runs sampled) + #startSpan with 100 attributes x 3,716 ops/sec ±1.92% (10 runs sampled) + #startSpan with SimpleSpanProcessor x 5,440 ops/sec ±39.90% (10 runs sampled) + #startSpan with BatchSpanProcessor x 2,284 ops/sec ±6.51% (10 runs sampled) + +Beginning NodeTracerRegistry Benchmark... + 7 tests completed. + + #startSpan x 81,777 ops/sec ±4.32% (10 runs sampled) + #startSpan:parent x 57,455 ops/sec ±3.87% (10 runs sampled) + #startSpan with attribute x 85,139 ops/sec ±4.09% (10 runs sampled) + #startSpan with 30 attributes x 38,240 ops/sec ±1.95% (10 runs sampled) + #startSpan with 100 attributes x 3,670 ops/sec ±6.85% (10 runs sampled) + #startSpan with SimpleSpanProcessor x 4,504 ops/sec ±37.04% (10 runs sampled) + #startSpan with BatchSpanProcessor x 1,847 ops/sec ±5.26% (10 runs sampled) + + +Beginning B3Format Benchmark... + 2 tests completed. + + #Inject x 5,569,330 ops/sec ±1.44% (10 runs sampled) + #Extract x 4,882,488 ops/sec ±3.72% (10 runs sampled) + +Beginning HttpTraceContext Benchmark... + 2 tests completed. + + #Inject x 13,423,892 ops/sec ±4.62% (10 runs sampled) + #Extract x 1,673,804 ops/sec ±2.29% (10 runs sampled) +``` \ No newline at end of file diff --git a/benchmark/tracer.js b/benchmark/tracer.js index 83700b0171d..d3ff4cfaf96 100644 --- a/benchmark/tracer.js +++ b/benchmark/tracer.js @@ -21,7 +21,8 @@ const setups = [ for (const setup of setups) { console.log(`Beginning ${setup.name} Benchmark...`); - const tracer = setup.registry.getTracer("benchmark"); + const registry = setup.registry; + const tracer = registry.getTracer("benchmark"); const suite = benchmark() .add('#startSpan', function () { const span = tracer.startSpan('op'); From ccc4a32a01d05555ccbe5d10fe19c22a4ec6d618 Mon Sep 17 00:00:00 2001 From: Mayur Kale Date: Mon, 13 Jan 2020 10:51:29 -0800 Subject: [PATCH 2/5] chore: update readme chore: update readme --- benchmark/README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/benchmark/README.md b/benchmark/README.md index d1c37b781bf..ea71b540f57 100644 --- a/benchmark/README.md +++ b/benchmark/README.md @@ -3,15 +3,15 @@ ## How to run To run your benchmark, just: -``` +```sh $ npm run bench ``` -If you're interested in writing benchmarking for other APIs, please write a benchmark in the `benchmark/index.js` module. Please refer to the `benchmark/tracer.js` or `benchmark/propagator.js` for more comprehensive examples. +> NOTE: If you're interested in writing benchmarking for other APIs, please write a benchmark in the `benchmark/index.js` module. Please refer to the `benchmark/tracer.js` or `benchmark/propagator.js` for more comprehensive examples. ## Results -# `0.3.3` release +### `v0.3.3` release ``` Beginning BasicTracerRegistry Benchmark... @@ -48,4 +48,4 @@ Beginning HttpTraceContext Benchmark... #Inject x 13,423,892 ops/sec ±4.62% (10 runs sampled) #Extract x 1,673,804 ops/sec ±2.29% (10 runs sampled) -``` \ No newline at end of file +``` From 3c324abd2b11931dbf3ae3ec5f89d4ee311e5358 Mon Sep 17 00:00:00 2001 From: Mayur Kale Date: Mon, 13 Jan 2020 10:56:35 -0800 Subject: [PATCH 3/5] chore: update readme chore: update readme --- benchmark/README.md | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/benchmark/README.md b/benchmark/README.md index ea71b540f57..c1d8a238814 100644 --- a/benchmark/README.md +++ b/benchmark/README.md @@ -7,7 +7,9 @@ To run your benchmark, just: $ npm run bench ``` -> NOTE: If you're interested in writing benchmarking for other APIs, please write a benchmark in the `benchmark/index.js` module. Please refer to the `benchmark/tracer.js` or `benchmark/propagator.js` for more comprehensive examples. +The minimum sample size is set to 10 to perform statistical analysis on benchmark, you can re-configure that in `benchmark.js`. + +> NOTE: If you're interested in writing benchmark for other APIs, please write a benchmark in the `benchmark/index.js` module. Please refer to the `benchmark/tracer.js` or `benchmark/propagator.js` for more comprehensive examples. ## Results From 97afbc431e99043ec536b0d276a978dc1d0ce0d8 Mon Sep 17 00:00:00 2001 From: Daniel Dyla Date: Fri, 17 Jan 2020 14:23:08 -0500 Subject: [PATCH 4/5] chore: update benchmarks --- benchmark/benchmark.js | 5 ++--- benchmark/index.js | 4 ++-- benchmark/propagator.js | 4 ++-- benchmark/tracer.js | 46 ++++++++++++++++++----------------------- 4 files changed, 26 insertions(+), 33 deletions(-) diff --git a/benchmark/benchmark.js b/benchmark/benchmark.js index cb71c03a873..8bc04a47b5a 100644 --- a/benchmark/benchmark.js +++ b/benchmark/benchmark.js @@ -4,10 +4,9 @@ const Benchmark = require('benchmark'); const benchmarks = require('beautify-benchmark'); Benchmark.options.maxTime = 0; -// @todo : Change it to between 50-100 or keep it random. -Benchmark.options.minSamples = 10; -module.exports = () => { +module.exports = (minSamples) => { + Benchmark.options.minSamples = minSamples; const suite = new Benchmark.Suite(); return suite diff --git a/benchmark/index.js b/benchmark/index.js index ebf5aff932b..745b93b75a6 100644 --- a/benchmark/index.js +++ b/benchmark/index.js @@ -3,5 +3,5 @@ const execSync = require('child_process').execSync; const exec = cmd => execSync(cmd, { stdio: [0, 1, 2] }); -exec('node benchmark/tracer'); -exec('node benchmark/propagator'); +exec('node benchmark/tracer.js'); +exec('node benchmark/propagator.js'); diff --git a/benchmark/propagator.js b/benchmark/propagator.js index f28a78a3026..0c5c137c26d 100644 --- a/benchmark/propagator.js +++ b/benchmark/propagator.js @@ -1,7 +1,7 @@ 'use strict'; const benchmark = require('./benchmark'); -const opentelemetry = require('@opentelemetry/core'); +const opentelemetry = require('../packages/opentelemetry-core'); const setups = [ { @@ -26,7 +26,7 @@ const setups = [ for (const setup of setups) { console.log(`Beginning ${setup.name} Benchmark...`); const propagator = setup.propagator; - const suite = benchmark() + const suite = benchmark(100) .add('#Inject', function () { propagator.inject({ traceId: 'd4cda95b652f4a1592b449d5929fda1b', diff --git a/benchmark/tracer.js b/benchmark/tracer.js index d3ff4cfaf96..e2a63a22ea6 100644 --- a/benchmark/tracer.js +++ b/benchmark/tracer.js @@ -1,11 +1,9 @@ 'use strict'; const benchmark = require('./benchmark'); -const opentelemetry = require('@opentelemetry/core'); -const { BasicTracerRegistry, BatchSpanProcessor, InMemorySpanExporter, SimpleSpanProcessor } = require('@opentelemetry/tracing'); -const { NodeTracerRegistry } = require('@opentelemetry/node'); +const opentelemetry = require('../packages/opentelemetry-core'); +const { BasicTracerRegistry, BatchSpanProcessor, InMemorySpanExporter, SimpleSpanProcessor } = require('../packages/opentelemetry-tracing'); -const exporter = new InMemorySpanExporter(); const logger = new opentelemetry.NoopLogger(); const setups = [ @@ -14,16 +12,23 @@ const setups = [ registry: new BasicTracerRegistry({ logger }) }, { - name: 'NodeTracerRegistry', - registry: new NodeTracerRegistry({ logger }) + name: 'BasicTracerRegistry with SimpleSpanProcessor', + registry: getRegistry(new SimpleSpanProcessor(new InMemorySpanExporter())) + }, + { + name: 'BasicTracerRegistry with BatchSpanProcessor', + registry: getRegistry(new BatchSpanProcessor(new InMemorySpanExporter())) + }, + { + name: 'NoopTracerRegistry', + registry: opentelemetry.getTracerRegistry() } ]; for (const setup of setups) { console.log(`Beginning ${setup.name} Benchmark...`); - const registry = setup.registry; - const tracer = registry.getTracer("benchmark"); - const suite = benchmark() + const tracer = setup.registry.getTracer("benchmark"); + const suite = benchmark(20) .add('#startSpan', function () { const span = tracer.startSpan('op'); span.end(); @@ -52,25 +57,14 @@ for (const setup of setups) { span.setAttribute('attr-key-' + j, 'attr-value-' + j); } span.end(); - }) - .add('#startSpan with SimpleSpanProcessor', function () { - const simpleSpanProcessor = new SimpleSpanProcessor(exporter); - - registry.addSpanProcessor(simpleSpanProcessor); - const span = tracer.startSpan('op'); - span.end(); - - simpleSpanProcessor.shutdown(); - }) - .add('#startSpan with BatchSpanProcessor', function () { - const batchSpanProcessor = new BatchSpanProcessor(exporter); - - registry.addSpanProcessor(batchSpanProcessor); - const span = tracer.startSpan('op'); - span.end(); - batchSpanProcessor.shutdown(); }); // run async suite.run({ async: false }); } +function getRegistry(processor) { + const registry = new BasicTracerRegistry({ logger }); + registry.addSpanProcessor(processor); + return registry; +} + From 5de4b8c32978baac5b99251261f3b772a5eab93a Mon Sep 17 00:00:00 2001 From: Mayur Kale Date: Wed, 22 Jan 2020 11:07:05 -0800 Subject: [PATCH 5/5] generate latest benchmark numbers --- benchmark/README.md | 56 ++++++++++++++++++++++++++++----------------- benchmark/tracer.js | 8 +++---- 2 files changed, 39 insertions(+), 25 deletions(-) diff --git a/benchmark/README.md b/benchmark/README.md index c1d8a238814..0f1e81a9fa2 100644 --- a/benchmark/README.md +++ b/benchmark/README.md @@ -16,38 +16,52 @@ The minimum sample size is set to 10 to perform statistical analysis on benchmar ### `v0.3.3` release ``` +Beginning NoopTracerRegistry Benchmark... + 5 tests completed. + + #startSpan x 731,516,636 ops/sec ±2.57% (20 runs sampled) + #startSpan:parent x 744,353,590 ops/sec ±3.03% (20 runs sampled) + #startSpan with attribute x 737,451,332 ops/sec ±3.75% (20 runs sampled) + #startSpan with 30 attributes x 1,658,688 ops/sec ±1.23% (20 runs sampled) + #startSpan with 100 attributes x 535,082 ops/sec ±1.55% (20 runs sampled) + Beginning BasicTracerRegistry Benchmark... - 7 tests completed. + 5 tests completed. + + #startSpan x 80,633 ops/sec ±3.57% (20 runs sampled) + #startSpan:parent x 56,228 ops/sec ±2.18% (20 runs sampled) + #startSpan with attribute x 86,710 ops/sec ±1.80% (20 runs sampled) + #startSpan with 30 attributes x 36,331 ops/sec ±1.29% (20 runs sampled) + #startSpan with 100 attributes x 3,549 ops/sec ±3.59% (20 runs sampled) + +Beginning BasicTracerRegistry with SimpleSpanProcessor Benchmark... + 5 tests completed. - #startSpan x 79,704 ops/sec ±4.46% (10 runs sampled) - #startSpan:parent x 55,975 ops/sec ±1.90% (10 runs sampled) - #startSpan with attribute x 84,479 ops/sec ±2.82% (10 runs sampled) - #startSpan with 30 attributes x 36,239 ops/sec ±2.67% (10 runs sampled) - #startSpan with 100 attributes x 3,716 ops/sec ±1.92% (10 runs sampled) - #startSpan with SimpleSpanProcessor x 5,440 ops/sec ±39.90% (10 runs sampled) - #startSpan with BatchSpanProcessor x 2,284 ops/sec ±6.51% (10 runs sampled) + #startSpan x 74,539 ops/sec ±4.49% (20 runs sampled) + #startSpan:parent x 48,953 ops/sec ±4.98% (20 runs sampled) + #startSpan with attribute x 79,686 ops/sec ±2.54% (20 runs sampled) + #startSpan with 30 attributes x 26,491 ops/sec ±13.68% (20 runs sampled) + #startSpan with 100 attributes x 2,464 ops/sec ±19.64% (20 runs sampled) -Beginning NodeTracerRegistry Benchmark... - 7 tests completed. +Beginning BasicTracerRegistry with BatchSpanProcessor Benchmark... + 5 tests completed. - #startSpan x 81,777 ops/sec ±4.32% (10 runs sampled) - #startSpan:parent x 57,455 ops/sec ±3.87% (10 runs sampled) - #startSpan with attribute x 85,139 ops/sec ±4.09% (10 runs sampled) - #startSpan with 30 attributes x 38,240 ops/sec ±1.95% (10 runs sampled) - #startSpan with 100 attributes x 3,670 ops/sec ±6.85% (10 runs sampled) - #startSpan with SimpleSpanProcessor x 4,504 ops/sec ±37.04% (10 runs sampled) - #startSpan with BatchSpanProcessor x 1,847 ops/sec ±5.26% (10 runs sampled) + #startSpan x 74,974 ops/sec ±3.57% (20 runs sampled) + #startSpan:parent x 42,390 ops/sec ±20.68% (20 runs sampled) + #startSpan with attribute x 76,497 ops/sec ±2.93% (20 runs sampled) + #startSpan with 30 attributes x 33,042 ops/sec ±2.03% (20 runs sampled) + #startSpan with 100 attributes x 3,459 ops/sec ±4.56% (20 runs sampled) Beginning B3Format Benchmark... 2 tests completed. - #Inject x 5,569,330 ops/sec ±1.44% (10 runs sampled) - #Extract x 4,882,488 ops/sec ±3.72% (10 runs sampled) + #Inject x 5,086,366 ops/sec ±3.18% (100 runs sampled) + #Extract x 4,859,557 ops/sec ±3.80% (100 runs sampled) Beginning HttpTraceContext Benchmark... 2 tests completed. - #Inject x 13,423,892 ops/sec ±4.62% (10 runs sampled) - #Extract x 1,673,804 ops/sec ±2.29% (10 runs sampled) + #Inject x 13,660,710 ops/sec ±1.84% (100 runs sampled) + #Extract x 1,692,010 ops/sec ±0.83% (100 runs sampled) ``` diff --git a/benchmark/tracer.js b/benchmark/tracer.js index e2a63a22ea6..a1a6c360cdf 100644 --- a/benchmark/tracer.js +++ b/benchmark/tracer.js @@ -7,6 +7,10 @@ const { BasicTracerRegistry, BatchSpanProcessor, InMemorySpanExporter, SimpleSpa const logger = new opentelemetry.NoopLogger(); const setups = [ + { + name: 'NoopTracerRegistry', + registry: opentelemetry.getTracerRegistry() + }, { name: 'BasicTracerRegistry', registry: new BasicTracerRegistry({ logger }) @@ -18,10 +22,6 @@ const setups = [ { name: 'BasicTracerRegistry with BatchSpanProcessor', registry: getRegistry(new BatchSpanProcessor(new InMemorySpanExporter())) - }, - { - name: 'NoopTracerRegistry', - registry: opentelemetry.getTracerRegistry() } ];