From 2d6f9a5c27803257635b4b008c5d3b0592478e1d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Donny/=EA=B0=95=EB=8F=99=EC=9C=A4?= Date: Wed, 15 Jan 2025 04:37:30 +0900 Subject: [PATCH] perf(bench): Run benchmark on a custom runner (#9877) **Description:** I configured one custom runner using /~https://github.com/myoung34/docker-github-actions-runner that has 16 cores and 64GB RAM. It's much quieter than a public GitHub runner, so I'll rely on this for profiling. --- .github/workflows/bench.yml | 127 +++++++++++---------- crates/swc_css_minifier/Cargo.toml | 4 - crates/swc_css_minifier/benches/full.rs | 63 ---------- scripts/bench/build-all-crates.sh | 36 ------ scripts/bench/build-crate.sh | 14 +++ scripts/bench/list-crates-with-bench.sh | 6 + scripts/cargo/get-workspace-crates-json.sh | 5 + 7 files changed, 93 insertions(+), 162 deletions(-) delete mode 100644 crates/swc_css_minifier/benches/full.rs delete mode 100755 scripts/bench/build-all-crates.sh create mode 100755 scripts/bench/build-crate.sh create mode 100755 scripts/bench/list-crates-with-bench.sh create mode 100755 scripts/cargo/get-workspace-crates-json.sh diff --git a/.github/workflows/bench.yml b/.github/workflows/bench.yml index aef159a795fa..5e2784983fe3 100644 --- a/.github/workflows/bench.yml +++ b/.github/workflows/bench.yml @@ -24,65 +24,27 @@ env: SKIP_YARN_COREPACK_CHECK: 1 jobs: - # list-crates: - # if: >- - # ${{ !contains(github.event.head_commit.message, 'chore: ') }} - # name: List crates - # runs-on: ubuntu-latest - # outputs: - # crates: ${{ steps.list-crates.outputs.crates }} - # steps: - # - uses: actions/checkout@v4 - - # - name: Install Rust - # uses: actions-rs/toolchain@v1 - # with: - # profile: minimal - - # - name: List crates - # id: list-crates - # run: echo "crates=$(./scripts/cargo/get-crates.sh)" >> $GITHUB_OUTPUT - - # bench-crate: - # name: Bench ${{ matrix.crate }} - # runs-on: ubuntu-latest - # needs: - # - list-crates - # strategy: - # fail-fast: false - # matrix: - # crate: ${{fromJson(needs.list-crates.outputs.crates)}} - # steps: - # - uses: actions/checkout@v4 - - # - name: Install Rust - # uses: actions-rs/toolchain@v1 - # with: - # profile: minimal - - # - uses: ./.github/actions/setup-node - - # - name: Setup rust toolchain, cache and cargo-codspeed binary - # uses: moonrepo/setup-rust@v0 - # with: - # channel: stable - # cache-target: release - # bins: cargo-codspeed - - # - name: Build the benchmark target(s) - # run: cargo codspeed build -p ${{ matrix.crate }} - - # - name: Run the benchmarks - # uses: CodSpeedHQ/action@v2 - # with: - # run: cargo codspeed run -p ${{ matrix.crate }} - # token: ${{ secrets.CODSPEED_TOKEN }} - - bench-all: - name: Bench everything + list-crates: if: >- ${{ !contains(github.event.head_commit.message, 'chore: ') }} + name: List crates to benchmark + runs-on: ubuntu-22.04 + outputs: + crates: ${{ steps.list-crates.outputs.crates }} + steps: + - uses: actions/checkout@v4 + + - name: List crates + id: list-crates + run: echo "crates=$(./scripts/bench/list-crates-with-bench.sh)" >> $GITHUB_OUTPUT + + build-crate: + name: Build benchmark for ${{ matrix.crate }} runs-on: ubuntu-22.04 + needs: list-crates + strategy: + matrix: + crate: ${{fromJson(needs.list-crates.outputs.crates)}} steps: - uses: actions/checkout@v4 @@ -91,7 +53,13 @@ jobs: with: profile: minimal - - uses: ./.github/actions/setup-node + - uses: Swatinem/rust-cache@v2 + # Some crates are too slow to build + if: matrix.crate == 'swc' + with: + shared-key: "bench-${{ matrix.crate }}" + key: "bench-${{ matrix.crate }}" + cache-all-crates: true - name: Install cargo-codspeed uses: taiki-e/install-action@v2 @@ -99,10 +67,51 @@ jobs: tool: cargo-codspeed@2.7.2 - name: Build the benchmark target(s) - run: ./scripts/bench/build-all-crates.sh + run: ./scripts/bench/build-crate.sh ${{ matrix.crate }} + + - name: Upload built artifacts + uses: actions/upload-artifact@v4 + with: + name: bench-${{ matrix.crate }} + path: target/codspeed + + bench-all: + name: Run benchmarks + needs: build-crate + runs-on: + - self-hosted + - linux + - x64 + if: >- + ${{ !contains(github.event.head_commit.message, 'chore: ') }} + steps: + - uses: actions/checkout@v4 + + - name: Install Rust + uses: actions-rs/toolchain@v1 + with: + profile: minimal + + - name: Install cargo-codspeed + uses: taiki-e/install-action@v2 + with: + tool: cargo-codspeed@2.7.2 + + - run: mkdir -p target/codspeed + + - name: Download built artifacts + uses: actions/download-artifact@v4 + with: + pattern: bench-* + path: target/codspeed + merge-multiple: true + + - run: ls -alR target/codspeed + + - run: chmod -R +x target/codspeed/* - name: Run the benchmarks - uses: CodSpeedHQ/action@v2 + uses: CodSpeedHQ/action@v3 with: run: cargo codspeed run token: ${{ secrets.CODSPEED_TOKEN }} diff --git a/crates/swc_css_minifier/Cargo.toml b/crates/swc_css_minifier/Cargo.toml index 2b609e505cce..75bf6080e4ea 100644 --- a/crates/swc_css_minifier/Cargo.toml +++ b/crates/swc_css_minifier/Cargo.toml @@ -29,7 +29,3 @@ swc_css_codegen = { version = "5.0.0", path = "../swc_css_codegen" } swc_css_parser = { version = "5.0.0", path = "../swc_css_parser" } swc_malloc = { version = "1.0.0", path = "../swc_malloc" } testing = { version = "5.0.0", path = "../testing" } - -[[bench]] -harness = false -name = "full" diff --git a/crates/swc_css_minifier/benches/full.rs b/crates/swc_css_minifier/benches/full.rs deleted file mode 100644 index 3d00261cf38f..000000000000 --- a/crates/swc_css_minifier/benches/full.rs +++ /dev/null @@ -1,63 +0,0 @@ -extern crate swc_malloc; - -use std::{fs::read_to_string, path::Path}; - -use codspeed_criterion_compat::{black_box, criterion_group, criterion_main, Criterion}; -use swc_common::{errors::HANDLER, FileName}; -use swc_css_ast::Stylesheet; -use swc_css_codegen::{writer::basic::BasicCssWriter, Emit}; -use swc_css_minifier::minify; -use swc_css_parser::parse_file; - -pub fn bench_files(c: &mut Criterion) { - let mut group = c.benchmark_group("css/minify/libraries"); - group.sample_size(10); - - let mut bench_file = |name: &str, path: &Path| { - let src = read_to_string(path).unwrap(); - - group.bench_function(name, |b| { - b.iter(|| { - // We benchmark full time, including time for creating cm, handler - run(&src) - }) - }); - }; - - bench_file( - "bootstrap", - Path::new("../../node_modules/bootstrap/dist/css/bootstrap.css"), - ); -} - -criterion_group!(files, bench_files); -criterion_main!(files); - -fn run(src: &str) { - testing::run_test2(false, |cm, handler| { - HANDLER.set(&handler, || { - let fm = cm.new_source_file(FileName::Anon.into(), src.into()); - - let mut errors = Vec::new(); - let mut ss: Stylesheet = - parse_file(&fm, None, Default::default(), &mut errors).unwrap(); - - minify(&mut ss, Default::default()); - - let mut buf = String::new(); - { - let wr = BasicCssWriter::new(&mut buf, None, Default::default()); - let mut generator = swc_css_codegen::CodeGenerator::new( - wr, - swc_css_codegen::CodegenConfig { minify: true }, - ); - - generator.emit(&ss).unwrap(); - } - - black_box(buf); - Ok(()) - }) - }) - .unwrap(); -} diff --git a/scripts/bench/build-all-crates.sh b/scripts/bench/build-all-crates.sh deleted file mode 100755 index 147ef5e29088..000000000000 --- a/scripts/bench/build-all-crates.sh +++ /dev/null @@ -1,36 +0,0 @@ -#!/usr/bin/env bash - -set -eu - -echo "Building all crates for codspeed" - -WS_CRATES=$(./scripts/cargo/list-crates.sh) - -# Get crate names -CRATE_NAMES=$(echo "$WS_CRATES" | jq -r '.name') - -for crate in $CRATE_NAMES; do - # If crate is swc_plugin_runner, skip it - if [[ $crate == "swc_plugin_runner" ]]; then - continue - fi - - crate_info=$(echo "$WS_CRATES" | jq -r 'select(.name == "'$crate'")') - bench_targets=$(echo $crate_info | jq -r '.targets[] | select(.kind | contains(["bench"]))') - - - # If crate has no benchmark target, skip it - if [[ -z $bench_targets ]]; then - echo "Skipping $crate because it has no benchmark target" - continue - fi - - # If crate has feature 'concurrent', build it with feature - if [[ $(./scripts/cargo/list-features.sh $crate) == *"concurrent"* ]]; then - echo "Building $crate with feature 'concurrent'" - cargo codspeed build -p $crate --features concurrent - else - echo "Building $crate" - cargo codspeed build -p $crate - fi -done diff --git a/scripts/bench/build-crate.sh b/scripts/bench/build-crate.sh new file mode 100755 index 000000000000..ef0c815dcf8e --- /dev/null +++ b/scripts/bench/build-crate.sh @@ -0,0 +1,14 @@ +#!/usr/bin/env bash + +set -eu + +crate=$1 + +# If crate has feature 'concurrent', build it with feature +if [[ $(./scripts/cargo/list-features.sh $crate) == *"concurrent"* ]]; then + echo "Building $crate with feature 'concurrent'" + cargo codspeed build -p $crate --features concurrent +else + echo "Building $crate" + cargo codspeed build -p $crate +fi \ No newline at end of file diff --git a/scripts/bench/list-crates-with-bench.sh b/scripts/bench/list-crates-with-bench.sh new file mode 100755 index 000000000000..8dd0d5f9d6a8 --- /dev/null +++ b/scripts/bench/list-crates-with-bench.sh @@ -0,0 +1,6 @@ +#!/usr/bin/env bash + +set -eu + +WS_CRATES=$(./scripts/cargo/get-workspace-crates-json.sh) +echo "$WS_CRATES" | jq -r -c '[.[] | select(.targets[] | .kind | contains(["bench"])) | .name] | sort | unique' | jq -r -c '[.[] | select(. != "swc_plugin_runner")]' diff --git a/scripts/cargo/get-workspace-crates-json.sh b/scripts/cargo/get-workspace-crates-json.sh new file mode 100755 index 000000000000..18cce3a9ed2b --- /dev/null +++ b/scripts/cargo/get-workspace-crates-json.sh @@ -0,0 +1,5 @@ +#!/usr/bin/env bash + +set -eu + +cargo metadata --format-version 1 --no-deps | jq -r -j '[.packages[] | select(.source == null and .name != "xtask")]' \ No newline at end of file