name: Benchmark Asterinas on: # In case of manual trigger, use workflow_dispatch workflow_dispatch: schedule: # Schedule to run on every day at 20:00 UTC (04:00 Beijing Time) - cron: '0 20 * * *' jobs: Benchmarks: runs-on: self-hosted strategy: matrix: benchmarks: - sysbench/cpu_lat - sysbench/thread_lat # Memory-related benchmarks - lmbench/mem_read_bw - lmbench/mem_write_bw - lmbench/mem_copy_bw - lmbench/mem_pagefault_lat - lmbench/mem_mmap_bw - lmbench/mem_mmap_lat # Process-related benchmarks - lmbench/process_getppid_lat - lmbench/process_ctx_lat - lmbench/process_fork_lat - lmbench/process_exec_lat - lmbench/process_shell_lat # Signal-related benchmarks - lmbench/signal_catch_lat - lmbench/signal_install_lat - lmbench/signal_prot_lat # IPC-related benchmarks - lmbench/pipe_lat - lmbench/pipe_bw - lmbench/fifo_lat - lmbench/semaphore_lat - lmbench/unix_lat - lmbench/unix_bw - lmbench/unix_connect_lat # Syscall-related benchmarks - lmbench/vfs_fstat_lat - lmbench/vfs_open_lat - lmbench/vfs_stat_lat - lmbench/vfs_write_lat - lmbench/vfs_read_lat - lmbench/vfs_select_lat - lmbench/vfs_fcntl_lat - lmbench/vfs_read_pagecache_bw # File-related benchmarks - lmbench/ramfs_create_delete_files_0k_ops - lmbench/ramfs_create_delete_files_10k_ops - lmbench/ramfs_copy_files_bw - lmbench/ext2_create_delete_files_0k_ops - lmbench/ext2_create_delete_files_10k_ops - lmbench/ext2_copy_files_bw - fio/ext2_seq_write_bw - fio/ext2_seq_read_bw - fio/ext2_seq_write_bw_no_iommu - fio/ext2_seq_read_bw_no_iommu # Loopback-related network benchmarks - lmbench/tcp_loopback_bw_128 - lmbench/tcp_loopback_bw_4k - lmbench/tcp_loopback_bw_64k - lmbench/tcp_loopback_lat - lmbench/tcp_loopback_connect_lat - lmbench/tcp_loopback_select_lat - lmbench/tcp_loopback_http_bw - lmbench/udp_loopback_lat # VirtIO-net-related network benchmarks - lmbench/tcp_virtio_bw_128 - lmbench/tcp_virtio_bw_64k - lmbench/tcp_virtio_connect_lat - lmbench/tcp_virtio_lat - lmbench/udp_virtio_lat - iperf3/tcp_virtio_bw # Scheduler-related benchmarks - hackbench/group8_smp1 # FIXME: hackbench panics on multi-core settings now. # - hackbench/group8_smp8 # - hackbench/group8_smp16 - schbench/smp1 - schbench/smp8 # Nginx benchmarks - nginx/http_req10k_conc1_bw - nginx/http_req10k_conc20_bw # Redis benchmarks - redis/ping_inline_100k_conc20_rps - redis/ping_mbulk_100k_conc20_rps - redis/get_100k_conc20_rps - redis/set_100k_conc20_rps # SQLite benchmarks - sqlite/ext2_benchmarks - sqlite/ramfs_benchmarks fail-fast: false timeout-minutes: 60 container: image: asterinas/asterinas:0.11.0 options: --device=/dev/kvm --privileged env: # Need to set up proxy since the self-hosted CI server is located in China, # which has poor network connection to the official Rust crate repositories. RUSTUP_DIST_SERVER: https://mirrors.ustc.edu.cn/rust-static RUSTUP_UPDATE_ROOT: https://mirrors.ustc.edu.cn/rust-static/rustup steps: - uses: actions/checkout@v4 - name: Set up the environment run: | chmod +x test/benchmark/bench_linux_and_aster.sh # Set up git due to the network issue on the self-hosted runner git config --global --add safe.directory /__w/asterinas/asterinas git config --global http.sslVerify false git config --global http.version HTTP/1.1 - name: Run benchmarks uses: nick-invision/retry@v3 # Retry the benchmark command in case of failure with: timeout_minutes: 20 max_attempts: 3 command: | make install_osdk bash test/benchmark/bench_linux_and_aster.sh "${{ matrix.benchmarks }}" BENCHMARK_ARTIFACT=results_$(echo "${{ matrix.benchmarks }}" | tr '/' '-') echo "BENCHMARK_ARTIFACT=$BENCHMARK_ARTIFACT" >> $GITHUB_ENV - name: Store benchmark results uses: actions/upload-artifact@v4 with: name: ${{ env.BENCHMARK_ARTIFACT }} if-no-files-found: error # Fail the benchmark job if no file is found. path: | result_*.json Matrix: runs-on: ubuntu-latest needs: Benchmarks # Must run after the Benchmarks job. if: always() # Always run regardless of whether the previous job was successful or not. outputs: benchmarks: ${{ steps.set-matrix.outputs.benchmarks }} steps: - name: Download results uses: actions/download-artifact@v4 with: pattern: results_* path: ./results merge-multiple: true - name: Set matrix for benchmark results id: set-matrix run: | benchmarks=$(ls results/result_*.json | sed 's/.*result_//' | sed 's/\.json//' | jq -R -s -c 'split("\n")[:-1]') echo benchmarks=$benchmarks >> $GITHUB_OUTPUT Results: runs-on: ubuntu-latest needs: Matrix if: always() # Always run regardless of whether the previous job was successful or not. strategy: matrix: benchmark: ${{ fromJson(needs.Matrix.outputs.benchmarks) }} max-parallel: 1 fail-fast: false steps: - uses: actions/checkout@v4 - name: Download Benchmark Results uses: actions/download-artifact@v4 with: pattern: results_* path: ./results merge-multiple: true - name: Set up benchmark configuration run: | BENCHMARK_DIR=$(echo ${{ matrix.benchmark }} | sed 's/-/\//g') BENCHMARK_SUITE=$(echo $BENCHMARK_DIR | awk -F'/' '{print $1}') BENCHMARK_NAME=$(basename "$BENCHMARK_DIR") BENCH_RESULT="test/benchmark/${BENCHMARK_DIR}/bench_result.json" [ -f "$BENCH_RESULT" ] || BENCH_RESULT="test/benchmark/${BENCHMARK_DIR}.json" ALERT_THRESHOLD=$(jq -r '.alert.threshold // "130%"' "$BENCH_RESULT") ALERT_TOOL=$(jq -r 'if .alert.bigger_is_better == true then "customBiggerIsBetter" else "customSmallerIsBetter" end' "$BENCH_RESULT") TITLE=$(jq -r '.chart.title // "Undefined"' "$BENCH_RESULT") DESCRIPTION=$(jq -r '.chart.description // "No description provided"' "$BENCH_RESULT") echo "BENCHMARK_SUITE=$BENCHMARK_SUITE" >> $GITHUB_ENV echo "BENCHMARK_NAME=$BENCHMARK_NAME" >> $GITHUB_ENV echo "ALERT_THRESHOLD=$ALERT_THRESHOLD" >> $GITHUB_ENV echo "ALERT_TOOL=$ALERT_TOOL" >> $GITHUB_ENV echo "TITLE=$TITLE" >> $GITHUB_ENV echo "DESCRIPTION=$DESCRIPTION" >> $GITHUB_ENV - name: Store benchmark results uses: asterinas/github-action-benchmark@v4 with: name: ${{ env.BENCHMARK_NAME }} tool: ${{ env.ALERT_TOOL }} output-file-path: results/result_${{ matrix.benchmark }}.json benchmark-data-dir-path: ${{ env.BENCHMARK_SUITE }} github-token: ${{ secrets.BENCHMARK_SECRET }} gh-repository: 'github.com/asterinas/benchmark' auto-push: true alert-threshold: ${{ env.ALERT_THRESHOLD }} comment-on-alert: true fail-on-alert: true chart-title: ${{ env.TITLE }} chart-description: ${{ env.DESCRIPTION }} summary-json-path: test/benchmark/${{ env.BENCHMARK_SUITE }}/summary.json