Skip to content

Commit

Permalink
chore: minor improvements to default benchmarks. (#2993)
Browse files Browse the repository at this point in the history 
* chore: minor improvements to default benchmarks.

* chore: lint issues fix.

* 🦉 Updates from OwlBot post-processor

See /~https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md

---------

Co-authored-by: Owl Bot <gcf-owl-bot[bot]@users.noreply.github.com>
  • Loading branch information
@arpan14 @gcf-owl-bot
arpan14 and gcf-owl-bot[bot] authored Apr 2, 2024
1 parent a8f1852 commit 54bef01
Show file tree
Hide file tree
Showing 2 changed files with 74 additions and 48 deletions.
64 changes: 58 additions & 6 deletions google-cloud-spanner/src/test/java/com/google/cloud/spanner/AbstractLatencyBenchmark.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,7 @@

package com.google.cloud.spanner;

import com.google.common.base.MoreObjects;
import java.time.Duration;
import java.util.ArrayList;
import java.util.Collections;
Expand All @@ -24,6 +25,52 @@

public abstract class AbstractLatencyBenchmark {

static final String SELECT_QUERY = "SELECT ID FROM FOO WHERE ID = @id";
static final String UPDATE_QUERY = "UPDATE FOO SET BAR=1 WHERE ID = @id";
static final String ID_COLUMN_NAME = "id";

/**
* Used to determine how many concurrent requests are allowed. For ex - To simulate a low QPS
* scenario, using 1 thread means there will be 1 request. Use a value > 1 to have concurrent
* requests.
*/
static final int PARALLEL_THREADS =
Integer.valueOf(
MoreObjects.firstNonNull(System.getenv("SPANNER_TEST_JMH_NUM_PARALLEL_THREADS"), "30"));

/**
* Total number of reads per test run for 1 thread. Increasing the value here will increase the
* duration of the benchmark. For ex - With PARALLEL_THREADS = 2, TOTAL_READS_PER_RUN = 200, there
* will be 400 read requests (200 on each thread).
*/
static final int TOTAL_READS_PER_RUN =
Integer.valueOf(
MoreObjects.firstNonNull(
System.getenv("SPANNER_TEST_JMH_NUM_READS_PER_THREAD"), "48000"));

/**
* Total number of writes per test run for 1 thread. Increasing the value here will increase the
* duration of the benchmark. For ex - With PARALLEL_THREADS = 2, TOTAL_WRITES_PER_RUN = 200,
* there will be 400 write requests (200 on each thread).
*/
static final int TOTAL_WRITES_PER_RUN =
Integer.valueOf(
MoreObjects.firstNonNull(
System.getenv("SPANNER_TEST_JMH_NUM_WRITES_PER_THREAD"), "4000"));

/**
* Number of requests which are used to initialise/warmup the benchmark. The latency number of
* these runs are ignored from the final reported results.
*/
static final int WARMUP_REQUEST_COUNT = 1;

/**
* Numbers of records in the sample table used in the benchmark. This is used in this benchmark to
* randomly choose a primary key and ensure that the reads are randomly distributed. This is done
* to ensure we don't end up reading/writing the same table record (leading to hot-spotting).
*/
static final int TOTAL_RECORDS = 1000000;

/** Utility to print latency numbers. It computes metrics such as Average, P50, P95 and P99. */
public void printResults(List<Duration> results) {
if (results == null) {
Expand All @@ -33,23 +80,23 @@ public void printResults(List<Duration> results) {
Collections.sort(orderedResults);
System.out.println();
System.out.printf("Total number of queries: %d\n", orderedResults.size());
System.out.printf("Avg: %fs\n", avg(results));
System.out.printf("P50: %fs\n", percentile(50, orderedResults));
System.out.printf("P95: %fs\n", percentile(95, orderedResults));
System.out.printf("P99: %fs\n", percentile(99, orderedResults));
System.out.printf("Avg: %fms\n", avg(results));
System.out.printf("P50: %fms\n", percentile(50, orderedResults));
System.out.printf("P95: %fms\n", percentile(95, orderedResults));
System.out.printf("P99: %fms\n", percentile(99, orderedResults));
}

private double percentile(int percentile, List<Duration> orderedResults) {
int index = percentile * orderedResults.size() / 100;
Duration value = orderedResults.get(index);
Double convertedValue = convertDurationToFractionInSeconds(value);
Double convertedValue = convertDurationToFractionInMilliSeconds(value);
return convertedValue;
}

/** Returns the average duration in seconds from a list of duration values. */
private double avg(List<Duration> results) {
return results.stream()
.collect(Collectors.averagingDouble(this::convertDurationToFractionInSeconds));
.collect(Collectors.averagingDouble(this::convertDurationToFractionInMilliSeconds));
}

private double convertDurationToFractionInSeconds(Duration duration) {
Expand All @@ -59,4 +106,9 @@ private double convertDurationToFractionInSeconds(Duration duration) {
double value = seconds + fraction;
return value;
}

private double convertDurationToFractionInMilliSeconds(Duration duration) {
long nanoseconds = duration.toNanos();
return nanoseconds / 1000000.0;
}
}
58 changes: 16 additions & 42 deletions google-cloud-spanner/src/test/java/com/google/cloud/spanner/DefaultBenchmark.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -18,6 +18,8 @@

import static com.google.cloud.spanner.BenchmarkingUtilityScripts.collectResults;
import static com.google.common.truth.Truth.assertThat;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNotNull;

import com.google.common.base.Stopwatch;
import com.google.common.util.concurrent.ListenableFuture;
Expand All @@ -37,6 +39,7 @@
import org.openjdk.jmh.annotations.Measurement;
import org.openjdk.jmh.annotations.Mode;
import org.openjdk.jmh.annotations.OutputTimeUnit;
import org.openjdk.jmh.annotations.Param;
import org.openjdk.jmh.annotations.Scope;
import org.openjdk.jmh.annotations.Setup;
import org.openjdk.jmh.annotations.State;
Expand All @@ -45,8 +48,8 @@

/**
* Benchmarks for measuring existing latencies of various APIs using the Java Client. The benchmarks
* are bound to the Maven profile `benchmark` and can be executed like this: <code> mvn clean test
* -DskipTests -Pbenchmark -Dbenchmark.name=DefaultBenchmark
* are bound to the Maven profile `benchmark` and can be executed like this: <code>
* mvn clean test -DskipTests -Pbenchmark -Dbenchmark.name=DefaultBenchmark
* </code> Test Table Schema :
*
* <p>CREATE TABLE FOO ( id INT64 NOT NULL, BAZ INT64, BAR INT64, ) PRIMARY KEY(id);
Expand All @@ -63,47 +66,9 @@
@Fork(value = 1, warmups = 0)
@Measurement(batchSize = 1, iterations = 1, timeUnit = TimeUnit.MILLISECONDS)
@OutputTimeUnit(TimeUnit.SECONDS)
@Warmup(iterations = 1)
@Warmup(iterations = 0)
public class DefaultBenchmark extends AbstractLatencyBenchmark {

private static final String SELECT_QUERY = "SELECT ID FROM FOO WHERE ID = @id";
private static final String UPDATE_QUERY = "UPDATE FOO SET BAR=1 WHERE ID = @id";
private static final String ID_COLUMN_NAME = "id";

/**
* Used to determine how many concurrent requests are allowed. For ex - To simulate a low QPS
* scenario, using 1 thread means there will be 1 request. Use a value > 1 to have concurrent
* requests.
*/
private static final int PARALLEL_THREADS = 1;

/**
* Total number of reads per test run for 1 thread. Increasing the value here will increase the
* duration of the benchmark. For ex - With PARALLEL_THREADS = 2, TOTAL_READS_PER_RUN = 200, there
* will be 400 read requests (200 on each thread).
*/
private static final int TOTAL_READS_PER_RUN = 12000;

/**
* Total number of writes per test run for 1 thread. Increasing the value here will increase the
* duration of the benchmark. For ex - With PARALLEL_THREADS = 2, TOTAL_WRITES_PER_RUN = 200,
* there will be 400 write requests (200 on each thread).
*/
private static final int TOTAL_WRITES_PER_RUN = 4000;

/**
* Number of requests which are used to initialise/warmup the benchmark. The latency number of
* these runs are ignored from the final reported results.
*/
private static final int WARMUP_REQUEST_COUNT = 1;

/**
* Numbers of records in the sample table used in the benchmark. This is used in this benchmark to
* randomly choose a primary key and ensure that the reads are randomly distributed. This is done
* to ensure we don't end up reading/writing the same table record (leading to hot-spotting).
*/
private static final int TOTAL_RECORDS = 1000000;

@State(Scope.Thread)
@AuxCounters(org.openjdk.jmh.annotations.AuxCounters.Type.EVENTS)
public static class BenchmarkState {
Expand All @@ -115,12 +80,20 @@ public static class BenchmarkState {
private Spanner spanner;
private DatabaseClientImpl client;

@Param({"100"})
int minSessions;

@Param({"400"})
int maxSessions;

@Setup(Level.Iteration)
public void setup() throws Exception {
SpannerOptions options =
SpannerOptions.newBuilder()
.setSessionPoolOption(
SessionPoolOptions.newBuilder()
.setMinSessions(minSessions)
.setMaxSessions(maxSessions)
.setWaitForMinSessions(org.threeten.bp.Duration.ofSeconds(20))
.build())
.setHost(SERVER_URL)
Expand Down Expand Up @@ -217,7 +190,8 @@ private java.time.Duration executeQuery(final BenchmarkState server) {

try (ResultSet rs = server.client.singleUse().executeQuery(getRandomisedReadStatement())) {
while (rs.next()) {
int count = rs.getColumnCount();
assertEquals(1, rs.getColumnCount());
assertNotNull(rs.getValue(0));
}
}
return watch.elapsed();
Expand Down

0 comments on commit 54bef01

Please sign in to comment.