name: Benchmark Asterinas
on:
  # In case of manual trigger, use workflow_dispatch
  workflow_dispatch:
  schedule:
    # Schedule to run on every day at 20:00 UTC (04:00 Beijing Time)
    - cron: '0 20 * * *'

jobs:
  Benchmarks:
    runs-on: self-hosted
    strategy:
      matrix:
        benchmark: 
          - sysbench/cpu_lat
          - sysbench/thread_lat
          # Memory-related benchmarks
          - lmbench/mem_read_bw
          - lmbench/mem_write_bw
          - lmbench/mem_copy_bw
          - lmbench/mem_pagefault_lat
          - lmbench/mem_mmap_bw
          - lmbench/mem_mmap_lat
          # Process-related benchmarks
          - lmbench/process_getppid_lat
          - lmbench/process_ctx_lat
          - lmbench/process_fork_lat
          - lmbench/process_exec_lat
          - lmbench/process_shell_lat
          # Signal-related benchmarks
          - lmbench/signal_catch_lat
          - lmbench/signal_install_lat
          - lmbench/signal_prot_lat
          # IPC-related benchmarks
          - lmbench/pipe_lat
          - lmbench/pipe_bw
          - lmbench/fifo_lat
          - lmbench/semaphore_lat
          - lmbench/unix_lat
          - lmbench/unix_bw
          - lmbench/unix_connect_lat
          # Syscall-related benchmarks
          - lmbench/vfs_fstat_lat
          - lmbench/vfs_open_lat
          - lmbench/vfs_stat_lat
          - lmbench/vfs_write_lat
          - lmbench/vfs_read_lat
          - lmbench/vfs_select_lat
          - lmbench/vfs_fcntl_lat
          - lmbench/vfs_read_pagecache_bw
          # File-related benchmarks
          - lmbench/ramfs_create_delete_files_0k_ops
          - lmbench/ramfs_create_delete_files_10k_ops
          - lmbench/ext2_create_delete_files_0k_ops
          - lmbench/ext2_create_delete_files_10k_ops
          - lmbench/ext2_copy_files_bw
          # Network-related benchmark
          - lmbench/tcp_loopback_bw
          - lmbench/tcp_loopback_lat
          - lmbench/tcp_loopback_connect_lat
          - lmbench/tcp_loopback_select_lat
          - lmbench/tcp_loopback_http_bw
          - lmbench/udp_loopback_lat
      fail-fast: false
    timeout-minutes: 60
    container: 
      image: asterinas/asterinas:0.8.3
      options: --device=/dev/kvm
    env:
      # Need to set up proxy since the self-hosted CI server is located in China,
      # which has poor network connection to the official Rust crate repositories.
      RUSTUP_DIST_SERVER: https://mirrors.ustc.edu.cn/rust-static
      RUSTUP_UPDATE_ROOT: https://mirrors.ustc.edu.cn/rust-static/rustup

    steps:
    - uses: actions/checkout@v2
    - name: Set up the environment
      run: |
        chmod +x test/benchmark/bench_linux_and_aster.sh
        # Set up git due to the network issue on the self-hosted runner
        git config --global --add safe.directory /__w/asterinas/asterinas
        git config --global http.sslVerify false
        git config --global http.version HTTP/1.1

    - name: Run benchmark
      uses: nick-invision/retry@v2 # Retry the benchmark command in case of failure
      with:
        timeout_minutes: 20
        max_attempts: 3
        command: |
          make install_osdk
          bash test/benchmark/bench_linux_and_aster.sh ${{ matrix.benchmark }}

    - name: Set up benchmark configuration
      run: |
        ALERT_THRESHOLD=$(jq -r '.alert_threshold' test/benchmark/${{ matrix.benchmark }}/config.json)
        ALERT_TOOL=$(jq -r '.alert_tool' test/benchmark/${{ matrix.benchmark }}/config.json)
        TITLE=$(jq -r '.title' test/benchmark/${{ matrix.benchmark }}/config.json)
        DESCRIPTION=$(jq -r '.description' test/benchmark/${{ matrix.benchmark }}/config.json)
        SHOW_IN_OVERVIEW=$(jq -r '.show_in_overview' test/benchmark/${{ matrix.benchmark }}/config.json)

        if [ "$ALERT_THRESHOLD" = "null" ]; then
          ALERT_THRESHOLD="130%"
        fi
        if [ "$ALERT_TOOL" = "null" ]; then
          ALERT_TOOL="customSmallerIsBetter"
        fi
        if [ "$TITLE" = "null" ]; then
          TITLE="${{ matrix.benchmark }}"
        fi
        if [ "$SHOW_IN_OVERVIEW" = "null" ]; then
          SHOW_IN_OVERVIEW="false"
        fi

        echo "ALERT_THRESHOLD=$ALERT_THRESHOLD" >> $GITHUB_ENV
        echo "ALERT_TOOL=$ALERT_TOOL" >> $GITHUB_ENV
        echo "TITLE=$TITLE" >> $GITHUB_ENV
        echo "DESCRIPTION=$DESCRIPTION" >> $GITHUB_ENV
        echo "SHOW_IN_OVERVIEW=$SHOW_IN_OVERVIEW" >> $GITHUB_ENV

        BENCHMARK_NAME=$(basename "${{ matrix.benchmark }}")
        echo "BENCHMARK_NAME=$BENCHMARK_NAME" >> $GITHUB_ENV

    - name: Store benchmark results
      uses: asterinas/github-action-benchmark@v3
      with:
        name: ${{ matrix.benchmark }}
        tool: ${{ env.ALERT_TOOL }}
        output-file-path: result_${{ env.BENCHMARK_NAME }}.json
        benchmark-data-dir-path: '' 
        github-token: ${{ secrets.BENCHMARK_SECRET }}
        gh-repository: 'github.com/asterinas/benchmark'
        auto-push: true
        alert-threshold: ${{ env.ALERT_THRESHOLD }} 
        comment-on-alert: true
        fail-on-alert: true
        chart-title: ${{ env.TITLE }}
        chart-description: ${{ env.DESCRIPTION }}
        show-in-overview: ${{ env.SHOW_IN_OVERVIEW }}