[browserstack benchmark tool] Enable code snippet benchmark (#6704)

FEATURE * benchmark code snippet * Update benchmark_models.js * upd * show code snippet in tool * typo
tensorflow · Aug 2, 2022 · e5c138e · e5c138e
1 parent 2dc7a77
commit e5c138e
Show file tree

Hide file tree

Showing 3 changed files with 78 additions and 28 deletions.
diff --git a/e2e/benchmarks/browserstack-benchmark/README.md b/e2e/benchmarks/browserstack-benchmark/README.md
@@ -33,6 +33,10 @@ The Multi-device benchmark tool can benchmark the performance (time, memory) of
   Then you can see `> Running socket on port: 8001` on your Command-line interface.
 
 3. Open http://localhost:8001/ and start to benchmark.
+  3.1 If you want to benchmark code snippet. Please update [`benchmarkCodeSnippet` ](/~https://github.com/tensorflow/tfjs/pull/6704/files#diff-a7c2ef12f0f2bc1a6cabb45bc9850aa68d10644cd2786e6505456e5537dccadbR92)with your code snippet before running `node app.js` and select `codeSnippet` in `model name`:
+<div style="text-align:center">
+  <img src="https://user-images.githubusercontent.com/40653845/182249695-021db9a7-e0ef-47e6-8110-ddf777b598a5.png" alt="drawing" height="300px"/>
+</div>
 4. When the benchmark is complete, you can see the benchmark results in the webpage, like:
 <div style="text-align:center">
   <img src="https://user-images.githubusercontent.com/40653845/90341914-a432f180-dfb8-11ea-841e-0d9078c6d50d.png" alt="drawing" height="300px"/>

diff --git a/e2e/benchmarks/browserstack-benchmark/benchmark_models.js b/e2e/benchmarks/browserstack-benchmark/benchmark_models.js
@@ -58,47 +58,87 @@ async function getBenchmarkSummary(timeInfo, memoryInfo, modelName = 'model') {
 
 const KARMA_SERVER = './base';
 
-describe('benchmark models', () => {
+async function benchmarkModel(benchmarkParameters) {
+  // Load the model.
+  const benchmark = benchmarks[benchmarkParameters.model];
+  const numRuns = benchmarkParameters.numRuns;
+  let model;
+  if (benchmarkParameters.model === 'custom') {
+    if (benchmarkParameters.modelUrl == null) {
+      throw new Error('Please provide model url for the custom model.');
+    }
+    model = await loadModelByUrl(benchmarkParameters.modelUrl);
+  } else {
+    model = await benchmark.load();
+  }
+
+  // Benchmark.
+  let timeInfo;
+  let memoryInfo;
+  if (benchmark.predictFunc != null) {
+    const predict = benchmark.predictFunc();
+    timeInfo = await timeInference(() => predict(model), numRuns);
+    memoryInfo = await profileInference(() => predict(model));
+  } else {
+    const input = generateInput(model);
+    timeInfo = await timeModelInference(model, input, numRuns);
+    memoryInfo = await profileModelInference(model, input);
+  }
+
+  return `<tfjs_benchmark>${
+      JSON.stringify({timeInfo, memoryInfo})}</tfjs_benchmark>`;
+}
+
+async function benchmarkCodeSnippet(benchmarkParameters) {
+  /* Please set up environments to run your code snippet here. */
+  /* Start */
+  const img = tf.randomUniform([1, 240, 240, 3], 0, 1000);
+  const filter = tf.randomUniform([3, 3, 3, 3], 0, 1000);
+  /* End */
+
+  /* Please put your code snippet to benchmark into the predict function. */
+  /* Start */
+  const predict = () => {
+    return tf.conv2d(img, filter, 1, 'same');
+  };
+  /* End */
+
+  // Warm up.
+  await timeInference(predict, 1);
+
+  // Benchmark code snippet.
+  timeInfo = await timeInference(predict, benchmarkParameters.numRuns);
+  memoryInfo = await profileInference(predict);
+
+  return `<tfjs_benchmark>${JSON.stringify({
+    timeInfo,
+    memoryInfo,
+    codeSnippet: predict.toString()
+  })}</tfjs_benchmark>`;
+}
+
+describe('BrowserStack benchmark', () => {
   let benchmarkParameters;
   beforeAll(async () => {
     jasmine.DEFAULT_TIMEOUT_INTERVAL = 1000000;
     const response = await fetch(`${KARMA_SERVER}/benchmark_parameters.json`);
     benchmarkParameters = await response.json();
   });
 
-  it(`benchmark model`, async () => {
+  it(`benchmark`, async () => {
     try {
+      // Setup benchmark environments.
       await tf.setBackend(benchmarkParameters.backend);
 
-      // Load the model.
-      const benchmark = benchmarks[benchmarkParameters.model];
-      const numRuns = benchmarkParameters.numRuns;
-      let model;
-      if (benchmarkParameters.model === 'custom') {
-        if (benchmarkParameters.modelUrl == null) {
-          throw new Error('Please provide model url for the custom model.');
-        }
-        model = await loadModelByUrl(benchmarkParameters.modelUrl);
-      } else {
-        model = await benchmark.load();
-      }
-
-      // Benchmark.
-      let timeInfo;
-      let memoryInfo;
-      if (benchmark.predictFunc != null) {
-        const predict = benchmark.predictFunc();
-        timeInfo = await timeInference(() => predict(model), numRuns);
-        memoryInfo = await profileInference(() => predict(model));
+      // Run benchmark and stringify results.
+      let resultStr;
+      if (benchmarkParameters.model === 'codeSnippet') {
+        resultStr = await benchmarkCodeSnippet(benchmarkParameters);
       } else {
-        const input = generateInput(model);
-        timeInfo = await timeModelInference(model, input, numRuns);
-        memoryInfo = await profileModelInference(model, input);
+        resultStr = await benchmarkModel(benchmarkParameters);
       }
 
       // Report results.
-      const resultStr = `<tfjs_benchmark>${
-          JSON.stringify({timeInfo, memoryInfo})}</tfjs_benchmark>`;
       console.log(resultStr);
     } catch (error) {
       console.log(`<tfjs_error>${error}</tfjs_error>`);

diff --git a/e2e/benchmarks/browserstack-benchmark/index.js b/e2e/benchmarks/browserstack-benchmark/index.js
@@ -536,6 +536,10 @@ function drawBenchmarkResultSummaryTable(benchmarkResult) {
 
   values.push(['Number of kernels', memoryInfo.kernels.length]);
 
+  if ('codeSnippet' in benchmarkResult) {
+    values.push(['Code snippet', benchmarkResult.codeSnippet]);
+  }
+
   const surface = {
     name: 'Benchmark Summary',
     tab: tabId,
@@ -632,7 +636,9 @@ function showModelSelection() {
   const modelFolder = gui.addFolder('Model');
   let modelUrlController = null;
 
-  modelFolder.add(state.benchmark, 'model', Object.keys(benchmarks))
+  modelFolder
+      .add(
+          state.benchmark, 'model', [...Object.keys(benchmarks), 'codeSnippet'])
       .name('model name')
       .onChange(async model => {
         if (model === 'custom') {