From 36841c50d41216ba33e7ceba7fff7918d89f5237 Mon Sep 17 00:00:00 2001 From: Fabing Li Date: Wed, 26 Jun 2024 20:07:44 +0800 Subject: [PATCH] Add benchmark CI for sysbench and getpid --- .github/workflows/benchmarks.yml | 64 ++++++++++ .gitignore | 3 + Makefile | 6 + regression/Makefile | 29 +++-- regression/benchmark/bench_linux_and_aster.sh | 111 ++++++++++++++++++ regression/benchmark/getpid/config.json | 5 + .../benchmark/getpid/result_template.json | 14 +++ regression/benchmark/getpid/run.sh | 9 ++ regression/benchmark/sysbench-cpu/config.json | 5 + .../sysbench-cpu/result_template.json | 14 +++ regression/benchmark/sysbench-cpu/run.sh | 16 +++ .../benchmark/sysbench-thread/config.json | 5 + .../sysbench-thread/result_template.json | 14 +++ regression/benchmark/sysbench-thread/run.sh | 16 +++ 14 files changed, 302 insertions(+), 9 deletions(-) create mode 100644 .github/workflows/benchmarks.yml create mode 100755 regression/benchmark/bench_linux_and_aster.sh create mode 100644 regression/benchmark/getpid/config.json create mode 100644 regression/benchmark/getpid/result_template.json create mode 100644 regression/benchmark/getpid/run.sh create mode 100644 regression/benchmark/sysbench-cpu/config.json create mode 100644 regression/benchmark/sysbench-cpu/result_template.json create mode 100755 regression/benchmark/sysbench-cpu/run.sh create mode 100644 regression/benchmark/sysbench-thread/config.json create mode 100644 regression/benchmark/sysbench-thread/result_template.json create mode 100755 regression/benchmark/sysbench-thread/run.sh diff --git a/.github/workflows/benchmarks.yml b/.github/workflows/benchmarks.yml new file mode 100644 index 00000000..3f174ea5 --- /dev/null +++ b/.github/workflows/benchmarks.yml @@ -0,0 +1,64 @@ +name: Benchmarks Test +on: + # In case of manual trigger, use workflow_dispatch + workflow_dispatch: + schedule: + # Schedule to run on every day at 00:00 UTC (08:00 CST) + - cron: '0 0 * * *' + +jobs: + Benchmarks: + runs-on: self-hosted + strategy: + matrix: + benchmark: [sysbench-cpu, sysbench-thread, getpid] + fail-fast: false + timeout-minutes: 60 + container: + image: asterinas/asterinas:0.5.1 + options: --device=/dev/kvm + env: + # Need to set up proxy since the self-hosted CI server is located in China, + # which has poor network connection to the official Rust crate repositories. + RUSTUP_DIST_SERVER: https://mirrors.ustc.edu.cn/rust-static + RUSTUP_UPDATE_ROOT: https://mirrors.ustc.edu.cn/rust-static/rustup + + steps: + - uses: actions/checkout@v2 + - name: Set up the environment + run: | + chmod +x regression/benchmark/bench_linux_and_aster.sh + # Set up git due to the network issue on the self-hosted runner + git config --global --add safe.directory /__w/asterinas/asterinas + git config --global http.sslVerify false + git config --global http.version HTTP/1.1 + + - name: Run benchmark + uses: nick-invision/retry@v2 # Retry the benchmark command in case of failure + with: + timeout_minutes: 20 + max_attempts: 3 + command: | + make install_osdk + bash regression/benchmark/bench_linux_and_aster.sh ${{ matrix.benchmark }} + on_retry_command: make clean + + - name: Prepare threshold values + run: | + echo "Configuring thresholds..." + ALERT_THRESHOLD=$(jq -r '.alert_threshold' regression/benchmark/${{ matrix.benchmark }}/config.json) + echo "ALERT_THRESHOLD=$ALERT_THRESHOLD" >> $GITHUB_ENV + + - name: Store benchmark results + uses: asterinas/github-action-benchmark@v1 + with: + name: ${{ matrix.benchmark }} Benchmark + tool: 'customSmallerIsBetter' + output-file-path: result_${{ matrix.benchmark }}.json + benchmark-data-dir-path: '' + github-token: ${{ secrets.BENCHMARK_SECRET }} + gh-repository: 'github.com/asterinas/benchmark' + auto-push: true + alert-threshold: ${{ env.ALERT_THRESHOLD }} + comment-on-alert: true + fail-on-alert: true diff --git a/.gitignore b/.gitignore index 0ed877a5..325d71d9 100644 --- a/.gitignore +++ b/.gitignore @@ -21,3 +21,6 @@ virtio-net.pcap # vscode launch config file .vscode/launch.json .vscode/launch.bak + +# benchmark results +/result_*.json diff --git a/Makefile b/Makefile index 10f7fd21..03d34381 100644 --- a/Makefile +++ b/Makefile @@ -2,6 +2,7 @@ # Global options. ARCH ?= x86_64 +BENCHMARK ?= none BOOT_METHOD ?= grub-rescue-iso BOOT_PROTOCOL ?= multiboot2 BUILD_SYSCALL_TEST ?= 0 @@ -39,6 +40,11 @@ export VSOCK=1 CARGO_OSDK_ARGS += --init-args="/regression/run_vsock_test.sh" endif +# If the BENCHMARK is set, we will run the benchmark in the kernel mode. +ifneq ($(BENCHMARK), none) +CARGO_OSDK_ARGS += --init-args="/benchmark/$(BENCHMARK)/run.sh" +endif + ifeq ($(RELEASE_LTO), 1) CARGO_OSDK_ARGS += --profile release-lto else ifeq ($(RELEASE), 1) diff --git a/regression/Makefile b/regression/Makefile index 8f9a53cf..f935281d 100644 --- a/regression/Makefile +++ b/regression/Makefile @@ -1,10 +1,12 @@ # SPDX-License-Identifier: MPL-2.0 +VDSO_DIR := /root/dependency +VDSO_LIB := $(VDSO_DIR)/vdso64.so MKFILE_PATH := $(abspath $(lastword $(MAKEFILE_LIST))) CUR_DIR := $(patsubst %/,%,$(dir $(MKFILE_PATH))) BUILD_DIR := $(CUR_DIR)/build -VDSO_DIR := $(BUILD_DIR)/linux_vdso INITRAMFS := $(BUILD_DIR)/initramfs +BENCHMARK_ENTRYPOINT := $(CUR_DIR)/benchmark/benchmark_entrypoint.sh INITRAMFS_FILELIST := $(BUILD_DIR)/initramfs.filelist INITRAMFS_IMAGE := $(BUILD_DIR)/initramfs.cpio.gz EXT2_IMAGE := $(BUILD_DIR)/ext2.img @@ -32,7 +34,7 @@ SYSCALL_TEST_DIR := $(INITRAMFS)/opt/syscall_test .PHONY: all all: build -$(INITRAMFS)/lib/x86_64-linux-gnu: | $(VDSO_DIR) +$(INITRAMFS)/lib/x86_64-linux-gnu: | $(VDSO_LIB) @mkdir -p $@ @cp -L /lib/x86_64-linux-gnu/libc.so.6 $@ @cp -L /lib/x86_64-linux-gnu/libstdc++.so.6 $@ @@ -45,14 +47,15 @@ $(INITRAMFS)/lib/x86_64-linux-gnu: | $(VDSO_DIR) @cp -L /lib/x86_64-linux-gnu/libdl.so.2 $@ @cp -L /lib/x86_64-linux-gnu/libz.so.1 $@ @cp -L /usr/local/benchmark/iperf/lib/libiperf.so.0 $@ - @cp -L $(VDSO_DIR)/vdso64.so $@ + @cp -L $(VDSO_LIB) $@ + +$(VDSO_LIB): | $(VDSO_DIR) + @# TODO: use a custom compiled vdso.so file in the future. + @wget https://raw.githubusercontent.com/asterinas/linux_vdso/2a6d2db/vdso64.so -O $@ $(VDSO_DIR): - @# TODO: use a custom compiled vdso.so file in the future. - @rm -rf $@ && mkdir -p $@ - @cd $@ && git clone https://github.com/asterinas/linux_vdso.git . - @cd $@ && git checkout 2a6d2db 2>/dev/null - + @mkdir -p $@ + $(INITRAMFS)/lib64: @mkdir -p $@ @cp -L /lib64/ld-linux-x86-64.so.2 $@ @@ -75,11 +78,19 @@ $(INITRAMFS)/usr/bin: | $(INITRAMFS)/bin $(INITRAMFS)/regression: @make --no-print-directory -C apps -$(INITRAMFS)/benchmark: +$(INITRAMFS)/benchmark: | $(INITRAMFS)/benchmark/bin + @cp -rf $(CUR_DIR)/benchmark/* $@ + @if [ -e $(BENCHMARK_ENTRYPOINT) ]; then \ + cp $(BENCHMARK_ENTRYPOINT) $@; \ + fi + +$(INITRAMFS)/benchmark/bin: @mkdir -p $@ @cp /usr/local/benchmark/sysbench/bin/sysbench $@ @cp /usr/local/benchmark/iperf/bin/iperf3 $@ @cp /usr/local/benchmark/membench/membench $@ + @# Replace the homebrewed getpid with a standard benchmark like UnixBench or LMbench. + @gcc -O2 $(CUR_DIR)/apps/getpid/getpid.c -o $@/getpid # Make necessary directories. $(INITRAMFS_EMPTY_DIRS): diff --git a/regression/benchmark/bench_linux_and_aster.sh b/regression/benchmark/bench_linux_and_aster.sh new file mode 100755 index 00000000..0c9e26bc --- /dev/null +++ b/regression/benchmark/bench_linux_and_aster.sh @@ -0,0 +1,111 @@ +#!/bin/bash + +# SPDX-License-Identifier: MPL-2.0 + +set -e + +# Ensure all dependencies are installed +command -v jq >/dev/null 2>&1 || { echo >&2 "jq is not installed. Aborting."; exit 1; } + +# Script directory +BENCHMARK_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" +# Kernel image +KERNEL_DIR="/root/dependency" +LINUX_KERNEL="${KERNEL_DIR}/vmlinuz" + +# Generate entrypoint script for Linux cases +generate_entrypoint_script() { + local benchmark="$1" + local init_script=$(cat < "${initramfs_entrypoint_script}" + chmod +x "${initramfs_entrypoint_script}" + + # TODO: enable nopti for Linux to make the comparison more fair + local qemu_cmd="/usr/local/qemu/bin/qemu-system-x86_64 \ + --no-reboot \ + -smp 1 \ + -m 8G \ + -machine q35,kernel-irqchip=split \ + -cpu Icelake-Server,+x2apic \ + --enable-kvm \ + -kernel ${LINUX_KERNEL} \ + -initrd ${BENCHMARK_DIR}/../build/initramfs.cpio.gz \ + -append 'console=ttyS0 rdinit=/benchmark/benchmark_entrypoint.sh' \ + -nographic \ + 2>&1 | tee ${linux_output}" + + if [ ! -f "${LINUX_KERNEL}" ]; then + echo "Downloading the Linux kernel image..." + mkdir -p "${KERNEL_DIR}" + curl -L -o "${LINUX_KERNEL}" \ + -H "Accept: application/vnd.github.v3.raw" \ + "https://api.github.com/repos/asterinas/linux_kernel/contents/vmlinuz-5.15.0-105-generic?ref=9e66d28" + fi + + echo "Running benchmark ${benchmark} on Linux and Asterinas..." + make run BENCHMARK=${benchmark} ENABLE_KVM=1 RELEASE=1 2>&1 | tee "${aster_output}" + eval "$qemu_cmd" + + echo "Parsing results..." + local LINUX_AVG ASTER_AVG + LINUX_AVG=$(awk "/${avg_pattern}/{print \$$avg_field}" "${linux_output}" | tr -d '\r') + ASTER_AVG=$(awk "/${avg_pattern}/{print \$$avg_field}" "${aster_output}" | tr -d '\r') + + if [ -z "${LINUX_AVG}" ] || [ -z "${ASTER_AVG}" ]; then + echo "Error: Failed to parse the average value from the benchmark output" >&2 + exit 1 + fi + + echo "Updating the result template with average values..." + jq --arg linux_avg "${LINUX_AVG}" --arg aster_avg "${ASTER_AVG}" \ + '(.[] | select(.extra == "linux_avg") | .value) |= $linux_avg | + (.[] | select(.extra == "aster_avg") | .value) |= $aster_avg' \ + "${result_template}" > "${result_file}" + + echo "Cleaning up..." + rm -f "${initramfs_entrypoint_script}" + rm -f "${linux_output}" + rm -f "${aster_output}" +} + + +# Main + +BENCHMARK="$1" +echo "Running benchmark ${BENCHMARK}..." +pwd +if [ ! -d "$BENCHMARK_DIR/$BENCHMARK" ]; then + echo "Error: Benchmark directory not found" >&2 + exit 1 +fi + +PATTERN=$(jq -r '.pattern' "$BENCHMARK_DIR/$BENCHMARK/config.json") +FIELD=$(jq -r '.field' "$BENCHMARK_DIR/$BENCHMARK/config.json") + +run_benchmark "$BENCHMARK" "$PATTERN" "$FIELD" + +echo "Benchmark completed successfully." diff --git a/regression/benchmark/getpid/config.json b/regression/benchmark/getpid/config.json new file mode 100644 index 00000000..d27458bb --- /dev/null +++ b/regression/benchmark/getpid/config.json @@ -0,0 +1,5 @@ +{ + "alert_threshold": "125%", + "pattern": "Syscall average latency:", + "field": "4" +} diff --git a/regression/benchmark/getpid/result_template.json b/regression/benchmark/getpid/result_template.json new file mode 100644 index 00000000..1f7acf76 --- /dev/null +++ b/regression/benchmark/getpid/result_template.json @@ -0,0 +1,14 @@ +[ + { + "name": "Average Syscall Latency on Linux", + "unit": "ns", + "value": 0, + "extra": "linux_avg" + }, + { + "name": "Average Syscall Latency on Asterinas", + "unit": "ns", + "value": 0, + "extra": "aster_avg" + } +] \ No newline at end of file diff --git a/regression/benchmark/getpid/run.sh b/regression/benchmark/getpid/run.sh new file mode 100644 index 00000000..152cb8da --- /dev/null +++ b/regression/benchmark/getpid/run.sh @@ -0,0 +1,9 @@ +#!/bin/sh + +# SPDX-License-Identifier: MPL-2.0 + +set -e + +echo "*** Running getpid ***" + +/benchmark/bin/getpid \ No newline at end of file diff --git a/regression/benchmark/sysbench-cpu/config.json b/regression/benchmark/sysbench-cpu/config.json new file mode 100644 index 00000000..6151709d --- /dev/null +++ b/regression/benchmark/sysbench-cpu/config.json @@ -0,0 +1,5 @@ +{ + "alert_threshold": "130%", + "pattern": "avg:", + "field": "NF" +} diff --git a/regression/benchmark/sysbench-cpu/result_template.json b/regression/benchmark/sysbench-cpu/result_template.json new file mode 100644 index 00000000..07359836 --- /dev/null +++ b/regression/benchmark/sysbench-cpu/result_template.json @@ -0,0 +1,14 @@ +[ + { + "name": "Average Execution Time per CPU on Linux", + "unit": "ms", + "value": 0, + "extra": "linux_avg" + }, + { + "name": "Average Execution Time per CPU on Asterinas", + "unit": "ms", + "value": 0, + "extra": "aster_avg" + } +] \ No newline at end of file diff --git a/regression/benchmark/sysbench-cpu/run.sh b/regression/benchmark/sysbench-cpu/run.sh new file mode 100755 index 00000000..259fd537 --- /dev/null +++ b/regression/benchmark/sysbench-cpu/run.sh @@ -0,0 +1,16 @@ +#!/bin/sh + +# SPDX-License-Identifier: MPL-2.0 + +set -e + +TEST_TIME=${1:-60} +TEST_THREADS=${2:-4} + +echo "*** Doing sysbench CPU test with ${TEST_THREADS} threads for ${TEST_TIME} seconds ***" + +/benchmark/bin/sysbench cpu \ + --threads=${TEST_THREADS} \ + --time=${TEST_TIME} \ + --cpu-max-prime=20000 run + \ No newline at end of file diff --git a/regression/benchmark/sysbench-thread/config.json b/regression/benchmark/sysbench-thread/config.json new file mode 100644 index 00000000..6151709d --- /dev/null +++ b/regression/benchmark/sysbench-thread/config.json @@ -0,0 +1,5 @@ +{ + "alert_threshold": "130%", + "pattern": "avg:", + "field": "NF" +} diff --git a/regression/benchmark/sysbench-thread/result_template.json b/regression/benchmark/sysbench-thread/result_template.json new file mode 100644 index 00000000..922e0726 --- /dev/null +++ b/regression/benchmark/sysbench-thread/result_template.json @@ -0,0 +1,14 @@ +[ + { + "name": "Average Execution Time per Thread on Linux", + "unit": "ms", + "value": 0, + "extra": "linux_avg" + }, + { + "name": "Average Execution Time per Thread on Asterinas", + "unit": "ms", + "value": 0, + "extra": "aster_avg" + } +] \ No newline at end of file diff --git a/regression/benchmark/sysbench-thread/run.sh b/regression/benchmark/sysbench-thread/run.sh new file mode 100755 index 00000000..06ef182a --- /dev/null +++ b/regression/benchmark/sysbench-thread/run.sh @@ -0,0 +1,16 @@ +#!/bin/sh + +# SPDX-License-Identifier: MPL-2.0 + +set -e + +TEST_TIME=${1:-60} +TEST_THREADS=${2:-200} + +echo "*** Doing sysbench with ${TEST_THREADS} threads for ${TEST_TIME} seconds ***" + +/benchmark/bin/sysbench threads \ + --threads=${TEST_THREADS} \ + --thread-yields=100 \ + --thread-locks=4 \ + --time=${TEST_TIME} run