From 84f12c7c6d56af0b72ebcec156cbccd17b61d6fb Mon Sep 17 00:00:00 2001 From: Siarhei Fedartsou Date: Fri, 26 Jul 2024 19:36:43 +0200 Subject: [PATCH] Add NodeJs bindings benchmarks (#7004) --- .github/workflows/osrm-backend.yml | 17 ++- package-lock.json | 13 +- package.json | 11 +- scripts/ci/bench.js | 211 +++++++++++++++++++++++++++++ scripts/ci/run_benchmarks.sh | 13 ++ 5 files changed, 258 insertions(+), 7 deletions(-) create mode 100644 scripts/ci/bench.js diff --git a/.github/workflows/osrm-backend.yml b/.github/workflows/osrm-backend.yml index 454e273f4ec..59329f539f2 100644 --- a/.github/workflows/osrm-backend.yml +++ b/.github/workflows/osrm-backend.yml @@ -672,6 +672,11 @@ jobs: key: v1-data-osm-pbf restore-keys: | v1-data-osm-pbf + - name: Use Node 20 + if: ${{ matrix.NODE_PACKAGE_TESTS_ONLY == 'ON' }} + uses: actions/setup-node@v4 + with: + node-version: 20 - name: Enable compiler cache uses: actions/cache@v4 with: @@ -722,18 +727,24 @@ jobs: path: base - name: Build Base Branch run: | + cd base + npm ci --ignore-scripts + cd .. mkdir base/build cd base/build - cmake -DENABLE_CONAN=ON -DCMAKE_BUILD_TYPE=Release .. + cmake -DENABLE_CONAN=ON -DCMAKE_BUILD_TYPE=Release -DENABLE_NODE_BINDINGS=ON .. make -j$(nproc) make -j$(nproc) benchmarks cd .. make -C test/data - name: Build PR Branch run: | + cd pr + npm ci --ignore-scripts + cd .. mkdir -p pr/build cd pr/build - cmake -DENABLE_CONAN=ON -DCMAKE_BUILD_TYPE=Release .. + cmake -DENABLE_CONAN=ON -DCMAKE_BUILD_TYPE=Release -DENABLE_NODE_BINDINGS=ON .. make -j$(nproc) make -j$(nproc) benchmarks cd .. @@ -745,6 +756,7 @@ jobs: run: | sudo mount -t tmpfs -o size=4g none /opt/benchmarks cp -rf pr/build /opt/benchmarks/build + cp -rf pr/lib /opt/benchmarks/lib mkdir -p /opt/benchmarks/test cp -rf pr/test/data /opt/benchmarks/test/data cp -rf pr/profiles /opt/benchmarks/profiles @@ -755,6 +767,7 @@ jobs: run: | sudo mount -t tmpfs -o size=4g none /opt/benchmarks cp -rf base/build /opt/benchmarks/build + cp -rf base/lib /opt/benchmarks/lib mkdir -p /opt/benchmarks/test cp -rf base/test/data /opt/benchmarks/test/data cp -rf base/profiles /opt/benchmarks/profiles diff --git a/package-lock.json b/package-lock.json index c5efc04e205..4731fa104f0 100644 --- a/package-lock.json +++ b/package-lock.json @@ -10,7 +10,8 @@ "hasInstallScript": true, "license": "BSD-2-Clause", "dependencies": { - "@mapbox/node-pre-gyp": "^1.0.11" + "@mapbox/node-pre-gyp": "^1.0.11", + "seedrandom": "^3.0.5" }, "devDependencies": { "@babel/cli": "^7.18.10", @@ -14652,6 +14653,11 @@ "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", "devOptional": true }, + "node_modules/seedrandom": { + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/seedrandom/-/seedrandom-3.0.5.tgz", + "integrity": "sha512-8OwmbklUNzwezjGInmZ+2clQmExQPvomqjL7LFqOYqtmuxRgQYqOD3mHaU+MvZn5FLUeVxVfQjwLZW/n/JFuqg==" + }, "node_modules/semver": { "version": "5.7.2", "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.2.tgz", @@ -30296,6 +30302,11 @@ "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==", "devOptional": true }, + "seedrandom": { + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/seedrandom/-/seedrandom-3.0.5.tgz", + "integrity": "sha512-8OwmbklUNzwezjGInmZ+2clQmExQPvomqjL7LFqOYqtmuxRgQYqOD3mHaU+MvZn5FLUeVxVfQjwLZW/n/JFuqg==" + }, "semver": { "version": "5.7.2", "resolved": "https://registry.npmjs.org/semver/-/semver-5.7.2.tgz", diff --git a/package.json b/package.json index 02eee8430f4..f36d90da593 100644 --- a/package.json +++ b/package.json @@ -4,7 +4,8 @@ "private": false, "description": "The Open Source Routing Machine is a high performance routing engine written in C++ designed to run on OpenStreetMap data.", "dependencies": { - "@mapbox/node-pre-gyp": "^1.0.11" + "@mapbox/node-pre-gyp": "^1.0.11", + "seedrandom": "^3.0.5" }, "browserify": { "transform": [ @@ -57,6 +58,7 @@ "jsonpath": "^1.1.1", "mkdirp": "^0.5.6", "node-addon-api": "^5.0.0", + "node-cmake": "^2.5.1", "node-timeout": "0.0.4", "polyline": "^0.2.0", "request": "^2.88.2", @@ -64,12 +66,13 @@ "tape": "^4.16.0", "turf": "^3.0.14", "uglify-js": "^3.17.0", - "xmlbuilder": "^4.2.1", - "node-cmake": "^2.5.1" + "xmlbuilder": "^4.2.1" }, "main": "lib/index.js", "binary": { - "napi_versions": [8], + "napi_versions": [ + 8 + ], "module_name": "node_osrm", "module_path": "./lib/binding_napi_v{napi_build_version}/", "host": "https://github.com", diff --git a/scripts/ci/bench.js b/scripts/ci/bench.js new file mode 100644 index 00000000000..71c3001c4e1 --- /dev/null +++ b/scripts/ci/bench.js @@ -0,0 +1,211 @@ +const fs = require('fs'); +const path = require('path'); +const readline = require('readline'); +const seedrandom = require('seedrandom'); + + +let RNG; + +class GPSData { + constructor(gpsTracesFilePath) { + this.tracks = {}; + this.coordinates = []; + this.trackIds = []; + this._loadGPSTraces(gpsTracesFilePath); + } + + _loadGPSTraces(gpsTracesFilePath) { + const expandedPath = path.resolve(gpsTracesFilePath); + const data = fs.readFileSync(expandedPath, 'utf-8'); + const lines = data.split('\n'); + const headers = lines[0].split(','); + + const latitudeIndex = headers.indexOf('Latitude'); + const longitudeIndex = headers.indexOf('Longitude'); + const trackIdIndex = headers.indexOf('TrackID'); + + for (let i = 1; i < lines.length; i++) { + if (lines[i].trim() === '') continue; + const row = lines[i].split(','); + + const latitude = parseFloat(row[latitudeIndex]); + const longitude = parseFloat(row[longitudeIndex]); + const trackId = row[trackIdIndex]; + + const coord = [longitude, latitude]; + this.coordinates.push(coord); + + if (!this.tracks[trackId]) { + this.tracks[trackId] = []; + } + this.tracks[trackId].push(coord); + } + + this.trackIds = Object.keys(this.tracks); + } + + getRandomCoordinate() { + const randomIndex = Math.floor(RNG() * this.coordinates.length); + return this.coordinates[randomIndex]; + } + + getRandomTrack() { + const randomIndex = Math.floor(RNG() * this.trackIds.length); + const trackId = this.trackIds[randomIndex]; + return this.tracks[trackId]; + } +}; + +async function runOSRMMethod(osrm, method, coordinates) { + const time = await new Promise((resolve, reject) => { + const startTime = process.hrtime(); + osrm[method]({coordinates}, (err, result) => { + if (err) { + if (['NoSegment', 'NoMatch', 'NoRoute', 'NoTrips'].includes(err.message)) { + resolve(null); + } else { + + reject(err); + } + } else { + const endTime = process.hrtime(startTime); + resolve(endTime[0] + endTime[1] / 1e9); + } + }); + }); + return time; +} + +async function nearest(osrm, gpsData) { + const times = []; + for (let i = 0; i < 1000; i++) { + const coord = gpsData.getRandomCoordinate(); + times.push(await runOSRMMethod(osrm, 'nearest', [coord])); + } + return times; +} + +async function route(osrm, gpsData) { + const times = []; + for (let i = 0; i < 1000; i++) { + const from = gpsData.getRandomCoordinate(); + const to = gpsData.getRandomCoordinate(); + + + times.push(await runOSRMMethod(osrm, 'route', [from, to])); + } + return times; +} + +async function table(osrm, gpsData) { + const times = []; + for (let i = 0; i < 250; i++) { + const numPoints = Math.floor(RNG() * 3) + 15; + const coordinates = []; + for (let i = 0; i < numPoints; i++) { + coordinates.push(gpsData.getRandomCoordinate()); + } + + + times.push(await runOSRMMethod(osrm, 'table', coordinates)); + } + return times; +} + +async function match(osrm, gpsData) { + const times = []; + for (let i = 0; i < 1000; i++) { + const numPoints = Math.floor(RNG() * 50) + 50; + const coordinates = gpsData.getRandomTrack().slice(0, numPoints); + + + times.push(await runOSRMMethod(osrm, 'match', coordinates)); + } + return times; +} + +async function trip(osrm, gpsData) { + const times = []; + for (let i = 0; i < 250; i++) { + const numPoints = Math.floor(RNG() * 2) + 5; + const coordinates = []; + for (let i = 0; i < numPoints; i++) { + coordinates.push(gpsData.getRandomCoordinate()); + } + + + times.push(await runOSRMMethod(osrm, 'trip', coordinates)); + } + return times; +} + +function bootstrapConfidenceInterval(data, numSamples = 1000, confidenceLevel = 0.95) { + let means = []; + let dataLength = data.length; + + for (let i = 0; i < numSamples; i++) { + let sample = []; + for (let j = 0; j < dataLength; j++) { + let randomIndex = Math.floor(RNG() * dataLength); + sample.push(data[randomIndex]); + } + let sampleMean = sample.reduce((a, b) => a + b, 0) / sample.length; + means.push(sampleMean); + } + + means.sort((a, b) => a - b); + let lowerBoundIndex = Math.floor((1 - confidenceLevel) / 2 * numSamples); + let upperBoundIndex = Math.floor((1 + confidenceLevel) / 2 * numSamples); + let mean = means.reduce((a, b) => a + b, 0) / means.length; + let lowerBound = means[lowerBoundIndex]; + let upperBound = means[upperBoundIndex]; + + return { mean: mean, lowerBound: lowerBound, upperBound: upperBound }; +} + +function calculateConfidenceInterval(data) { + let { mean, lowerBound, upperBound } = bootstrapConfidenceInterval(data); + let bestValue = Math.max(...data); + let errorMargin = (upperBound - lowerBound) / 2; + + return { mean, errorMargin, bestValue }; +} + +async function main() { + const args = process.argv.slice(2); + + const {OSRM} = require(args[0]); + const path = args[1]; + const algorithm = args[2].toUpperCase(); + const method = args[3]; + const gpsTracesFilePath = args[4]; + const iterations = parseInt(args[5]); + + const gpsData = new GPSData(gpsTracesFilePath); + const osrm = new OSRM({path, algorithm}); + + + const functions = { + route: route, + table: table, + nearest: nearest, + match: match, + trip: trip + }; + const func = functions[method]; + if (!func) { + throw new Error('Unknown method'); + } + const allTimes = []; + for (let i = 0; i < iterations; i++) { + RNG = seedrandom(42); + allTimes.push((await func(osrm, gpsData)).filter(t => t !== null)); + } + + const opsPerSec = allTimes.map(times => times.length / times.reduce((a, b) => a + b, 0)); + const { mean, errorMargin, bestValue } = calculateConfidenceInterval(opsPerSec); + console.log(`Ops: ${mean.toFixed(1)} ± ${errorMargin.toFixed(1)} ops/s. Best: ${bestValue.toFixed(1)} ops/s`); + +} + +main(); diff --git a/scripts/ci/run_benchmarks.sh b/scripts/ci/run_benchmarks.sh index 0d0324d13ac..5b092471d33 100755 --- a/scripts/ci/run_benchmarks.sh +++ b/scripts/ci/run_benchmarks.sh @@ -53,6 +53,7 @@ function run_benchmarks_for_folder { mkdir -p $RESULTS_FOLDER BENCHMARKS_FOLDER="$BINARIES_FOLDER/src/benchmarks" + echo "Running match-bench MLD" $BENCHMARKS_FOLDER/match-bench "$FOLDER/test/data/mld/monaco.osrm" mld > "$RESULTS_FOLDER/match_mld.bench" echo "Running match-bench CH" @@ -81,6 +82,18 @@ function run_benchmarks_for_folder { echo "Running osrm-contract" measure_peak_ram_and_time "$BINARIES_FOLDER/osrm-contract $FOLDER/data.osrm" "$RESULTS_FOLDER/osrm_contract.bench" + + for ALGORITHM in ch mld; do + for BENCH in nearest table trip route match; do + echo "Running node $BENCH $ALGORITHM" + START=$(date +%s.%N) + node $SCRIPTS_FOLDER/scripts/ci/bench.js $FOLDER/lib/binding/node_osrm.node $FOLDER/data.osrm $ALGORITHM $BENCH $GPS_TRACES > "$RESULTS_FOLDER/node_${BENCH}_${ALGORITHM}.bench" 5 + END=$(date +%s.%N) + DIFF=$(echo "$END - $START" | bc) + echo "Took: ${DIFF}s" + done + done + for ALGORITHM in ch mld; do for BENCH in nearest table trip route match; do echo "Running random $BENCH $ALGORITHM"