name: Benchmark Asterinas
on:
  # In case of manual trigger, use workflow_dispatch
  workflow_dispatch:
  schedule:
    # Schedule to run on every day at 20:00 UTC (04:00 Beijing Time)
    - cron: '0 20 * * *'

jobs:
  Benchmarks:
    runs-on: self-hosted
    strategy:
      matrix:
        benchmarks: 
          - sysbench/cpu_lat
          - sysbench/thread_lat
          # Memory-related benchmarks
          - lmbench/mem_read_bw
          - lmbench/mem_write_bw
          - lmbench/mem_copy_bw
          - lmbench/mem_pagefault_lat
          - lmbench/mem_mmap_bw
          - lmbench/mem_mmap_lat
          # Process-related benchmarks
          - lmbench/process_getppid_lat
          - lmbench/process_ctx_lat
          - lmbench/process_fork_lat
          - lmbench/process_exec_lat
          - lmbench/process_shell_lat
          # Signal-related benchmarks
          - lmbench/signal_catch_lat
          - lmbench/signal_install_lat
          - lmbench/signal_prot_lat
          # IPC-related benchmarks
          - lmbench/pipe_lat
          - lmbench/pipe_bw
          - lmbench/fifo_lat
          - lmbench/semaphore_lat
          - lmbench/unix_lat
          - lmbench/unix_bw
          - lmbench/unix_connect_lat
          # Syscall-related benchmarks
          - lmbench/vfs_fstat_lat
          - lmbench/vfs_open_lat
          - lmbench/vfs_stat_lat
          - lmbench/vfs_write_lat
          - lmbench/vfs_read_lat
          - lmbench/vfs_select_lat
          - lmbench/vfs_fcntl_lat
          - lmbench/vfs_read_pagecache_bw
          # File-related benchmarks
          - lmbench/ramfs_create_delete_files_0k_ops
          - lmbench/ramfs_create_delete_files_10k_ops
          - lmbench/ramfs_copy_files_bw
          - lmbench/ext2_create_delete_files_0k_ops
          - lmbench/ext2_create_delete_files_10k_ops
          - lmbench/ext2_copy_files_bw
          - fio/ext2_seq_write_bw
          - fio/ext2_seq_read_bw
          - fio/ext2_seq_write_bw_no_iommu
          - fio/ext2_seq_read_bw_no_iommu
          # Loopback-related network benchmarks
          - lmbench/tcp_loopback_bw_128
          - lmbench/tcp_loopback_bw_4k
          - lmbench/tcp_loopback_bw_64k
          - lmbench/tcp_loopback_lat
          - lmbench/tcp_loopback_connect_lat
          - lmbench/tcp_loopback_select_lat
          - lmbench/tcp_loopback_http_bw
          - lmbench/udp_loopback_lat
          # VirtIO-net-related network benchmarks
          - lmbench/tcp_virtio_bw_128
          - lmbench/tcp_virtio_bw_64k
          - lmbench/tcp_virtio_connect_lat
          - lmbench/tcp_virtio_lat
          - lmbench/udp_virtio_lat 
          - iperf3/tcp_virtio_bw
          # Scheduler-related benchmarks
          - hackbench/group8_smp1
          # FIXME: hackbench panics on multi-core settings now.
          # - hackbench/group8_smp8
          # - hackbench/group8_smp16
          - schbench/smp1
          - schbench/smp8
          # Nginx benchmarks
          - nginx/http_req10k_conc1_bw
          - nginx/http_req10k_conc20_bw
          - nginx/http_file4KB_bw
          - nginx/http_file8KB_bw
          - nginx/http_file16KB_bw
          - nginx/http_file32KB_bw
          - nginx/http_file64KB_bw
          # Redis benchmarks
          - redis/ping_inline_100k_conc20_rps
          - redis/ping_mbulk_100k_conc20_rps
          - redis/get_100k_conc20_rps
          - redis/set_100k_conc20_rps
          # SQLite benchmarks
          - sqlite/ext2_benchmarks
          - sqlite/ramfs_benchmarks
          # Memcached benchmarks
          - memcached/t8_conc32_window10k
          - memcached/t8_conc32_window20k
          - memcached/t16_conc64_window10k
      fail-fast: false
    timeout-minutes: 60
    container: 
      image: asterinas/asterinas:0.14.0
      options: --device=/dev/kvm --privileged
    env:
      # Need to set up proxy since the self-hosted CI server is located in China,
      # which has poor network connection to the official Rust crate repositories.
      RUSTUP_DIST_SERVER: https://mirrors.ustc.edu.cn/rust-static
      RUSTUP_UPDATE_ROOT: https://mirrors.ustc.edu.cn/rust-static/rustup

    steps:
      - uses: actions/checkout@v4
      - name: Set up the environment
        run: |
          chmod +x test/benchmark/bench_linux_and_aster.sh
          # Set up git due to the network issue on the self-hosted runner
          git config --global --add safe.directory /__w/asterinas/asterinas
          git config --global http.sslVerify false
          git config --global http.version HTTP/1.1

      - name: Run benchmarks
        uses: nick-invision/retry@v3 # Retry the benchmark command in case of failure
        with:
          timeout_minutes: 20
          max_attempts: 3
          command: |
            make install_osdk
            bash test/benchmark/bench_linux_and_aster.sh "${{ matrix.benchmarks }}"
            BENCHMARK_ARTIFACT=results_$(echo "${{ matrix.benchmarks }}" | tr '/' '-')
            echo "BENCHMARK_ARTIFACT=$BENCHMARK_ARTIFACT" >> $GITHUB_ENV

      - name: Store benchmark results
        uses: actions/upload-artifact@v4
        with:
          name: ${{ env.BENCHMARK_ARTIFACT }}
          if-no-files-found: error # Fail the benchmark job if no file is found.
          path: |
            result_*.json

  Matrix:
    runs-on: ubuntu-latest
    needs: Benchmarks # Must run after the Benchmarks job.
    if: always() # Always run regardless of whether the previous job was successful or not.
    outputs:
      benchmarks: ${{ steps.set-matrix.outputs.benchmarks }}
    steps:
      - name: Download results
        uses: actions/download-artifact@v4
        with:
          pattern: results_*
          path: ./results
          merge-multiple: true

      - name: Set matrix for benchmark results
        id: set-matrix
        run: |
          benchmarks=$(ls results/result_*.json | sed 's/.*result_//' | sed 's/\.json//' | jq -R -s -c 'split("\n")[:-1]')
          echo benchmarks=$benchmarks >> $GITHUB_OUTPUT

  Results:
    runs-on: ubuntu-latest
    needs: Matrix
    if: always() # Always run regardless of whether the previous job was successful or not.
    strategy:
      matrix:
        benchmark: ${{ fromJson(needs.Matrix.outputs.benchmarks) }}
      max-parallel: 1
      fail-fast: false

    steps:
      - uses: actions/checkout@v4
      - name: Download Benchmark Results
        uses: actions/download-artifact@v4
        with:
          pattern: results_*
          path: ./results
          merge-multiple: true
      
      - name: Set up the environment
        run: |
          sudo apt-get update && sudo apt-get install -y yq

      - name: Set up benchmark configuration
        run: |
          BENCHMARK_DIR=$(echo ${{ matrix.benchmark }} | sed 's/-/\//g')
          BENCHMARK_SUITE=$(echo $BENCHMARK_DIR | awk -F'/' '{print $1}')
          BENCHMARK_NAME=$(echo "$BENCHMARK_DIR" | sed -E 's|^[^/]+/||; s|/bench_results||g; s|/|_|g')
          BENCH_RESULT="test/benchmark/${BENCHMARK_DIR}/bench_result.yaml"
          [ -f "$BENCH_RESULT" ] || BENCH_RESULT="test/benchmark/${BENCHMARK_DIR}.yaml"
          ALERT_THRESHOLD=$(yq -r '.alert.threshold // "130%"' "$BENCH_RESULT")
          ALERT_TOOL=$(yq -r 'if (.alert.bigger_is_better == true) then "customBiggerIsBetter" else "customSmallerIsBetter" end' "$BENCH_RESULT")
          TITLE=$(yq -r '.chart.title // "Undefined"' "$BENCH_RESULT")
          DESCRIPTION=$(yq -r '.chart.description // "No description provided"' "$BENCH_RESULT")
          echo "BENCHMARK_SUITE=$BENCHMARK_SUITE" >> $GITHUB_ENV
          echo "BENCHMARK_NAME=$BENCHMARK_NAME" >> $GITHUB_ENV
          echo "ALERT_THRESHOLD=$ALERT_THRESHOLD" >> $GITHUB_ENV
          echo "ALERT_TOOL=$ALERT_TOOL" >> $GITHUB_ENV
          echo "TITLE=$TITLE" >> $GITHUB_ENV
          echo "DESCRIPTION=$DESCRIPTION" >> $GITHUB_ENV
          yq . test/benchmark/$BENCHMARK_SUITE/summary.yaml > test/benchmark/$BENCHMARK_SUITE/summary.json

      - name: Store benchmark results
        uses: asterinas/github-action-benchmark@v4
        with:
          name: ${{ env.BENCHMARK_NAME }}
          tool: ${{ env.ALERT_TOOL }}
          output-file-path: results/result_${{ matrix.benchmark }}.json
          benchmark-data-dir-path: ${{ env.BENCHMARK_SUITE }}
          github-token: ${{ secrets.BENCHMARK_SECRET }}
          gh-repository: 'github.com/asterinas/benchmark'
          auto-push: true
          alert-threshold: ${{ env.ALERT_THRESHOLD }}
          comment-on-alert: true
          fail-on-alert: true
          chart-title: ${{ env.TITLE }}
          chart-description: ${{ env.DESCRIPTION }}
          summary-json-path: test/benchmark/${{ env.BENCHMARK_SUITE }}/summary.json
          ref: ${{ github.sha }}
