mirror of
https://github.com/asterinas/asterinas.git
synced 2025-06-08 12:56:48 +00:00
223 lines
8.5 KiB
YAML
223 lines
8.5 KiB
YAML
name: Benchmark Asterinas
|
|
on:
|
|
# In case of manual trigger, use workflow_dispatch
|
|
workflow_dispatch:
|
|
schedule:
|
|
# Schedule to run on every day at 20:00 UTC (04:00 Beijing Time)
|
|
- cron: '0 20 * * *'
|
|
|
|
jobs:
|
|
Benchmarks:
|
|
runs-on: self-hosted
|
|
strategy:
|
|
matrix:
|
|
benchmarks:
|
|
- sysbench/cpu_lat
|
|
- sysbench/thread_lat
|
|
# Memory-related benchmarks
|
|
- lmbench/mem_read_bw
|
|
- lmbench/mem_write_bw
|
|
- lmbench/mem_copy_bw
|
|
- lmbench/mem_pagefault_lat
|
|
- lmbench/mem_mmap_bw
|
|
- lmbench/mem_mmap_lat
|
|
# Process-related benchmarks
|
|
- lmbench/process_getppid_lat
|
|
- lmbench/process_ctx_lat
|
|
- lmbench/process_fork_lat
|
|
- lmbench/process_exec_lat
|
|
- lmbench/process_shell_lat
|
|
# Signal-related benchmarks
|
|
- lmbench/signal_catch_lat
|
|
- lmbench/signal_install_lat
|
|
- lmbench/signal_prot_lat
|
|
# IPC-related benchmarks
|
|
- lmbench/pipe_lat
|
|
- lmbench/pipe_bw
|
|
- lmbench/fifo_lat
|
|
- lmbench/semaphore_lat
|
|
- lmbench/unix_lat
|
|
- lmbench/unix_bw
|
|
- lmbench/unix_connect_lat
|
|
# Syscall-related benchmarks
|
|
- lmbench/vfs_fstat_lat
|
|
- lmbench/vfs_open_lat
|
|
- lmbench/vfs_stat_lat
|
|
- lmbench/vfs_write_lat
|
|
- lmbench/vfs_read_lat
|
|
- lmbench/vfs_select_lat
|
|
- lmbench/vfs_fcntl_lat
|
|
- lmbench/vfs_read_pagecache_bw
|
|
# File-related benchmarks
|
|
- lmbench/ramfs_create_delete_files_0k_ops
|
|
- lmbench/ramfs_create_delete_files_10k_ops
|
|
- lmbench/ramfs_copy_files_bw
|
|
- lmbench/ext2_create_delete_files_0k_ops
|
|
- lmbench/ext2_create_delete_files_10k_ops
|
|
- lmbench/ext2_copy_files_bw
|
|
- fio/ext2_seq_write_bw
|
|
- fio/ext2_seq_read_bw
|
|
- fio/ext2_seq_write_bw_no_iommu
|
|
- fio/ext2_seq_read_bw_no_iommu
|
|
# Loopback-related network benchmarks
|
|
- lmbench/tcp_loopback_bw_128
|
|
- lmbench/tcp_loopback_bw_4k
|
|
- lmbench/tcp_loopback_bw_64k
|
|
- lmbench/tcp_loopback_lat
|
|
- lmbench/tcp_loopback_connect_lat
|
|
- lmbench/tcp_loopback_select_lat
|
|
- lmbench/tcp_loopback_http_bw
|
|
- lmbench/udp_loopback_lat
|
|
# VirtIO-net-related network benchmarks
|
|
- lmbench/tcp_virtio_bw_128
|
|
- lmbench/tcp_virtio_bw_64k
|
|
- lmbench/tcp_virtio_connect_lat
|
|
- lmbench/tcp_virtio_lat
|
|
- lmbench/udp_virtio_lat
|
|
- iperf3/tcp_virtio_bw
|
|
# Scheduler-related benchmarks
|
|
- hackbench/group8_smp1
|
|
# FIXME: hackbench panics on multi-core settings now.
|
|
# - hackbench/group8_smp8
|
|
# - hackbench/group8_smp16
|
|
- schbench/smp1
|
|
- schbench/smp8
|
|
# Nginx benchmarks
|
|
- nginx/http_req10k_conc1_bw
|
|
- nginx/http_req10k_conc20_bw
|
|
- nginx/http_file4KB_bw
|
|
- nginx/http_file8KB_bw
|
|
- nginx/http_file16KB_bw
|
|
- nginx/http_file32KB_bw
|
|
- nginx/http_file64KB_bw
|
|
# Redis benchmarks
|
|
- redis/ping_inline_100k_conc20_rps
|
|
- redis/ping_mbulk_100k_conc20_rps
|
|
- redis/get_100k_conc20_rps
|
|
- redis/set_100k_conc20_rps
|
|
# SQLite benchmarks
|
|
- sqlite/ext2_benchmarks
|
|
- sqlite/ramfs_benchmarks
|
|
# Memcached benchmarks
|
|
- memcached/t8_conc32_window10k
|
|
- memcached/t8_conc32_window20k
|
|
- memcached/t16_conc64_window10k
|
|
fail-fast: false
|
|
timeout-minutes: 60
|
|
container:
|
|
image: asterinas/asterinas:0.11.1
|
|
options: --device=/dev/kvm --privileged
|
|
env:
|
|
# Need to set up proxy since the self-hosted CI server is located in China,
|
|
# which has poor network connection to the official Rust crate repositories.
|
|
RUSTUP_DIST_SERVER: https://mirrors.ustc.edu.cn/rust-static
|
|
RUSTUP_UPDATE_ROOT: https://mirrors.ustc.edu.cn/rust-static/rustup
|
|
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
- name: Set up the environment
|
|
run: |
|
|
chmod +x test/benchmark/bench_linux_and_aster.sh
|
|
# Set up git due to the network issue on the self-hosted runner
|
|
git config --global --add safe.directory /__w/asterinas/asterinas
|
|
git config --global http.sslVerify false
|
|
git config --global http.version HTTP/1.1
|
|
|
|
- name: Run benchmarks
|
|
uses: nick-invision/retry@v3 # Retry the benchmark command in case of failure
|
|
with:
|
|
timeout_minutes: 20
|
|
max_attempts: 3
|
|
command: |
|
|
make install_osdk
|
|
bash test/benchmark/bench_linux_and_aster.sh "${{ matrix.benchmarks }}"
|
|
BENCHMARK_ARTIFACT=results_$(echo "${{ matrix.benchmarks }}" | tr '/' '-')
|
|
echo "BENCHMARK_ARTIFACT=$BENCHMARK_ARTIFACT" >> $GITHUB_ENV
|
|
|
|
- name: Store benchmark results
|
|
uses: actions/upload-artifact@v4
|
|
with:
|
|
name: ${{ env.BENCHMARK_ARTIFACT }}
|
|
if-no-files-found: error # Fail the benchmark job if no file is found.
|
|
path: |
|
|
result_*.json
|
|
|
|
Matrix:
|
|
runs-on: ubuntu-latest
|
|
needs: Benchmarks # Must run after the Benchmarks job.
|
|
if: always() # Always run regardless of whether the previous job was successful or not.
|
|
outputs:
|
|
benchmarks: ${{ steps.set-matrix.outputs.benchmarks }}
|
|
steps:
|
|
- name: Download results
|
|
uses: actions/download-artifact@v4
|
|
with:
|
|
pattern: results_*
|
|
path: ./results
|
|
merge-multiple: true
|
|
|
|
- name: Set matrix for benchmark results
|
|
id: set-matrix
|
|
run: |
|
|
benchmarks=$(ls results/result_*.json | sed 's/.*result_//' | sed 's/\.json//' | jq -R -s -c 'split("\n")[:-1]')
|
|
echo benchmarks=$benchmarks >> $GITHUB_OUTPUT
|
|
|
|
Results:
|
|
runs-on: ubuntu-latest
|
|
needs: Matrix
|
|
if: always() # Always run regardless of whether the previous job was successful or not.
|
|
strategy:
|
|
matrix:
|
|
benchmark: ${{ fromJson(needs.Matrix.outputs.benchmarks) }}
|
|
max-parallel: 1
|
|
fail-fast: false
|
|
|
|
steps:
|
|
- uses: actions/checkout@v4
|
|
- name: Download Benchmark Results
|
|
uses: actions/download-artifact@v4
|
|
with:
|
|
pattern: results_*
|
|
path: ./results
|
|
merge-multiple: true
|
|
|
|
- name: Set up the environment
|
|
run: |
|
|
sudo apt-get update && sudo apt-get install -y yq
|
|
|
|
- name: Set up benchmark configuration
|
|
run: |
|
|
BENCHMARK_DIR=$(echo ${{ matrix.benchmark }} | sed 's/-/\//g')
|
|
BENCHMARK_SUITE=$(echo $BENCHMARK_DIR | awk -F'/' '{print $1}')
|
|
BENCHMARK_NAME=$(echo "$BENCHMARK_DIR" | sed -E 's|^[^/]+/||; s|/bench_results||g; s|/|_|g')
|
|
BENCH_RESULT="test/benchmark/${BENCHMARK_DIR}/bench_result.yaml"
|
|
[ -f "$BENCH_RESULT" ] || BENCH_RESULT="test/benchmark/${BENCHMARK_DIR}.yaml"
|
|
ALERT_THRESHOLD=$(yq -r '.alert.threshold // "130%"' "$BENCH_RESULT")
|
|
ALERT_TOOL=$(yq -r 'if (.alert.bigger_is_better == true) then "customBiggerIsBetter" else "customSmallerIsBetter" end' "$BENCH_RESULT")
|
|
TITLE=$(yq -r '.chart.title // "Undefined"' "$BENCH_RESULT")
|
|
DESCRIPTION=$(yq -r '.chart.description // "No description provided"' "$BENCH_RESULT")
|
|
echo "BENCHMARK_SUITE=$BENCHMARK_SUITE" >> $GITHUB_ENV
|
|
echo "BENCHMARK_NAME=$BENCHMARK_NAME" >> $GITHUB_ENV
|
|
echo "ALERT_THRESHOLD=$ALERT_THRESHOLD" >> $GITHUB_ENV
|
|
echo "ALERT_TOOL=$ALERT_TOOL" >> $GITHUB_ENV
|
|
echo "TITLE=$TITLE" >> $GITHUB_ENV
|
|
echo "DESCRIPTION=$DESCRIPTION" >> $GITHUB_ENV
|
|
yq . test/benchmark/$BENCHMARK_SUITE/summary.yaml > test/benchmark/$BENCHMARK_SUITE/summary.json
|
|
|
|
- name: Store benchmark results
|
|
uses: asterinas/github-action-benchmark@v4
|
|
with:
|
|
name: ${{ env.BENCHMARK_NAME }}
|
|
tool: ${{ env.ALERT_TOOL }}
|
|
output-file-path: results/result_${{ matrix.benchmark }}.json
|
|
benchmark-data-dir-path: ${{ env.BENCHMARK_SUITE }}
|
|
github-token: ${{ secrets.BENCHMARK_SECRET }}
|
|
gh-repository: 'github.com/asterinas/benchmark'
|
|
auto-push: true
|
|
alert-threshold: ${{ env.ALERT_THRESHOLD }}
|
|
comment-on-alert: true
|
|
fail-on-alert: true
|
|
chart-title: ${{ env.TITLE }}
|
|
chart-description: ${{ env.DESCRIPTION }}
|
|
summary-json-path: test/benchmark/${{ env.BENCHMARK_SUITE }}/summary.json
|