Add benchmark CI for sysbench and getpid

.github/workflows/benchmarks.yml

@@ -0,0 +1,64 @@
+name: Benchmarks Test
+on:
+  # In case of manual trigger, use workflow_dispatch
+  workflow_dispatch:
+  schedule:
+    # Schedule to run on every day at 00:00 UTC (08:00 CST)
+    - cron: '0 0 * * *'
+
+jobs:
+  Benchmarks:
+    runs-on: self-hosted
+    strategy:
+      matrix:
+        benchmark: [sysbench-cpu, sysbench-thread, getpid]
+      fail-fast: false
+    timeout-minutes: 60
+    container: 
+      image: asterinas/asterinas:0.5.1
+      options: --device=/dev/kvm
+    env:
+      # Need to set up proxy since the self-hosted CI server is located in China,
+      # which has poor network connection to the official Rust crate repositories.
+      RUSTUP_DIST_SERVER: https://mirrors.ustc.edu.cn/rust-static
+      RUSTUP_UPDATE_ROOT: https://mirrors.ustc.edu.cn/rust-static/rustup
+
+    steps:
+    - uses: actions/checkout@v2
+    - name: Set up the environment
+      run: |
+        chmod +x regression/benchmark/bench_linux_and_aster.sh
+        # Set up git due to the network issue on the self-hosted runner
+        git config --global --add safe.directory /__w/asterinas/asterinas
+        git config --global http.sslVerify false
+        git config --global http.version HTTP/1.1
+
+    - name: Run benchmark
+      uses: nick-invision/retry@v2 # Retry the benchmark command in case of failure
+      with:
+        timeout_minutes: 20
+        max_attempts: 3
+        command: |
+          make install_osdk
+          bash regression/benchmark/bench_linux_and_aster.sh ${{ matrix.benchmark }}
+        on_retry_command: make clean
+
+    - name: Prepare threshold values
+      run: |
+        echo "Configuring thresholds..."
+        ALERT_THRESHOLD=$(jq -r '.alert_threshold' regression/benchmark/${{ matrix.benchmark }}/config.json)
+        echo "ALERT_THRESHOLD=$ALERT_THRESHOLD" >> $GITHUB_ENV
+
+    - name: Store benchmark results
+      uses: asterinas/github-action-benchmark@v1
+      with:
+        name: ${{ matrix.benchmark }} Benchmark
+        tool: 'customSmallerIsBetter'
+        output-file-path: result_${{ matrix.benchmark }}.json
+        benchmark-data-dir-path: '' 
+        github-token: ${{ secrets.BENCHMARK_SECRET }}
+        gh-repository: 'github.com/asterinas/benchmark'
+        auto-push: true
+        alert-threshold: ${{ env.ALERT_THRESHOLD }} 
+        comment-on-alert: true
+        fail-on-alert: true

.gitignore

@@ -21,3 +21,6 @@ virtio-net.pcap
 # vscode launch config file
 .vscode/launch.json
 .vscode/launch.bak
+
+# benchmark results
+/result_*.json

Makefile

@@ -2,6 +2,7 @@
 
 # Global options.
 ARCH ?= x86_64
+BENCHMARK ?= none
 BOOT_METHOD ?= grub-rescue-iso
 BOOT_PROTOCOL ?= multiboot2
 BUILD_SYSCALL_TEST ?= 0
@@ -39,6 +40,11 @@ export VSOCK=1
 CARGO_OSDK_ARGS += --init-args="/regression/run_vsock_test.sh"
 endif
 
+# If the BENCHMARK is set, we will run the benchmark in the kernel mode.
+ifneq ($(BENCHMARK), none)
+CARGO_OSDK_ARGS += --init-args="/benchmark/$(BENCHMARK)/run.sh"
+endif
+
 ifeq ($(RELEASE_LTO), 1)
 CARGO_OSDK_ARGS += --profile release-lto
 else ifeq ($(RELEASE), 1)

regression/Makefile

@@ -1,10 +1,12 @@
 # SPDX-License-Identifier: MPL-2.0
 
+VDSO_DIR := /root/dependency
+VDSO_LIB := $(VDSO_DIR)/vdso64.so
 MKFILE_PATH := $(abspath $(lastword $(MAKEFILE_LIST)))
 CUR_DIR := $(patsubst %/,%,$(dir $(MKFILE_PATH)))
 BUILD_DIR := $(CUR_DIR)/build
-VDSO_DIR := $(BUILD_DIR)/linux_vdso
 INITRAMFS := $(BUILD_DIR)/initramfs
+BENCHMARK_ENTRYPOINT := $(CUR_DIR)/benchmark/benchmark_entrypoint.sh
 INITRAMFS_FILELIST := $(BUILD_DIR)/initramfs.filelist
 INITRAMFS_IMAGE := $(BUILD_DIR)/initramfs.cpio.gz
 EXT2_IMAGE := $(BUILD_DIR)/ext2.img
@@ -32,7 +34,7 @@ SYSCALL_TEST_DIR := $(INITRAMFS)/opt/syscall_test
 .PHONY: all
 all: build
 
-$(INITRAMFS)/lib/x86_64-linux-gnu: | $(VDSO_DIR)
+$(INITRAMFS)/lib/x86_64-linux-gnu: | $(VDSO_LIB)
 	@mkdir -p $@
 	@cp -L /lib/x86_64-linux-gnu/libc.so.6 $@
 	@cp -L /lib/x86_64-linux-gnu/libstdc++.so.6 $@
@@ -45,14 +47,15 @@ $(INITRAMFS)/lib/x86_64-linux-gnu: | $(VDSO_DIR)
 	@cp -L /lib/x86_64-linux-gnu/libdl.so.2 $@
 	@cp -L /lib/x86_64-linux-gnu/libz.so.1 $@
 	@cp -L /usr/local/benchmark/iperf/lib/libiperf.so.0 $@
-	@cp -L $(VDSO_DIR)/vdso64.so $@
+	@cp -L $(VDSO_LIB) $@
+
+$(VDSO_LIB): | $(VDSO_DIR)
+	@# TODO: use a custom compiled vdso.so file in the future.
+	@wget https://raw.githubusercontent.com/asterinas/linux_vdso/2a6d2db/vdso64.so -O $@ 
 
 $(VDSO_DIR):
-	@# TODO: use a custom compiled vdso.so file in the future.
-	@rm -rf $@ && mkdir -p $@
-	@cd $@ && git clone https://github.com/asterinas/linux_vdso.git .
-	@cd $@ && git checkout 2a6d2db 2>/dev/null
-
+	@mkdir -p $@
+	
 $(INITRAMFS)/lib64:
 	@mkdir -p $@
 	@cp -L /lib64/ld-linux-x86-64.so.2 $@
@@ -75,11 +78,19 @@ $(INITRAMFS)/usr/bin: | $(INITRAMFS)/bin
 $(INITRAMFS)/regression:
 	@make --no-print-directory -C apps
 
-$(INITRAMFS)/benchmark:
+$(INITRAMFS)/benchmark: | $(INITRAMFS)/benchmark/bin
+	@cp -rf $(CUR_DIR)/benchmark/* $@
+	@if [ -e $(BENCHMARK_ENTRYPOINT) ]; then \
+		cp $(BENCHMARK_ENTRYPOINT) $@; \
+	fi
+
+$(INITRAMFS)/benchmark/bin:
 	@mkdir -p $@
 	@cp /usr/local/benchmark/sysbench/bin/sysbench $@
 	@cp /usr/local/benchmark/iperf/bin/iperf3 $@
 	@cp /usr/local/benchmark/membench/membench $@
+	@# Replace the homebrewed getpid with a standard benchmark like UnixBench or LMbench.
+	@gcc -O2 $(CUR_DIR)/apps/getpid/getpid.c -o $@/getpid
 
 # Make necessary directories.
 $(INITRAMFS_EMPTY_DIRS):

regression/benchmark/bench_linux_and_aster.sh

@@ -0,0 +1,111 @@
+#!/bin/bash
+
+# SPDX-License-Identifier: MPL-2.0
+
+set -e
+
+# Ensure all dependencies are installed
+command -v jq >/dev/null 2>&1 || { echo >&2 "jq is not installed. Aborting."; exit 1; }
+
+# Script directory
+BENCHMARK_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)"
+# Kernel image 
+KERNEL_DIR="/root/dependency"
+LINUX_KERNEL="${KERNEL_DIR}/vmlinuz"
+
+# Generate entrypoint script for Linux cases
+generate_entrypoint_script() {
+    local benchmark="$1"
+    local init_script=$(cat <<EOF
+#!/bin/sh
+
+echo "Running ${benchmark}"
+chmod +x /benchmark/${benchmark}/run.sh
+/benchmark/${benchmark}/run.sh
+
+poweroff -f
+EOF
+)
+    echo "$init_script"
+}
+
+run_benchmark() {
+    local benchmark="$1"
+    local avg_pattern="$2"
+    local avg_field="$3"
+
+    local linux_output="${BENCHMARK_DIR}/linux_output.txt"
+    local aster_output="${BENCHMARK_DIR}/aster_output.txt"
+    local result_template="${BENCHMARK_DIR}/${benchmark}/result_template.json"
+    local result_file="result_${benchmark}.json"
+
+    # Entrypoint script for initramfs
+    local initramfs_entrypoint_script="${BENCHMARK_DIR}/benchmark_entrypoint.sh"
+    generate_entrypoint_script "${benchmark}" > "${initramfs_entrypoint_script}"
+    chmod +x "${initramfs_entrypoint_script}"
+        
+    # TODO: enable nopti for Linux to make the comparison more fair
+    local qemu_cmd="/usr/local/qemu/bin/qemu-system-x86_64 \
+        --no-reboot \
+        -smp 1 \
+        -m 8G \
+        -machine q35,kernel-irqchip=split \
+        -cpu Icelake-Server,+x2apic \
+        --enable-kvm \
+        -kernel ${LINUX_KERNEL} \
+        -initrd ${BENCHMARK_DIR}/../build/initramfs.cpio.gz \
+        -append 'console=ttyS0 rdinit=/benchmark/benchmark_entrypoint.sh' \
+        -nographic \
+        2>&1 | tee ${linux_output}" 
+
+    if [ ! -f "${LINUX_KERNEL}" ]; then
+        echo "Downloading the Linux kernel image..."
+        mkdir -p "${KERNEL_DIR}"
+        curl -L -o "${LINUX_KERNEL}" \
+            -H "Accept: application/vnd.github.v3.raw" \
+            "https://api.github.com/repos/asterinas/linux_kernel/contents/vmlinuz-5.15.0-105-generic?ref=9e66d28"
+    fi
+
+    echo "Running benchmark ${benchmark} on Linux and Asterinas..."
+    make run BENCHMARK=${benchmark} ENABLE_KVM=1 RELEASE=1 2>&1 | tee "${aster_output}"
+    eval "$qemu_cmd"
+
+    echo "Parsing results..."
+    local LINUX_AVG ASTER_AVG
+    LINUX_AVG=$(awk "/${avg_pattern}/{print \$$avg_field}" "${linux_output}" | tr -d '\r')
+    ASTER_AVG=$(awk "/${avg_pattern}/{print \$$avg_field}" "${aster_output}" | tr -d '\r')
+
+    if [ -z "${LINUX_AVG}" ] || [ -z "${ASTER_AVG}" ]; then
+        echo "Error: Failed to parse the average value from the benchmark output" >&2
+        exit 1
+    fi
+
+    echo "Updating the result template with average values..."
+    jq --arg linux_avg "${LINUX_AVG}" --arg aster_avg "${ASTER_AVG}" \
+        '(.[] | select(.extra == "linux_avg") | .value) |= $linux_avg |
+         (.[] | select(.extra == "aster_avg") | .value) |= $aster_avg' \
+        "${result_template}" > "${result_file}"
+
+    echo "Cleaning up..."
+    rm -f "${initramfs_entrypoint_script}"
+    rm -f "${linux_output}"
+    rm -f "${aster_output}"
+}
+
+
+# Main
+
+BENCHMARK="$1"
+echo "Running benchmark ${BENCHMARK}..."
+pwd
+if [ ! -d "$BENCHMARK_DIR/$BENCHMARK" ]; then
+    echo "Error: Benchmark directory not found" >&2
+    exit 1
+fi
+
+PATTERN=$(jq -r '.pattern' "$BENCHMARK_DIR/$BENCHMARK/config.json")
+FIELD=$(jq -r '.field' "$BENCHMARK_DIR/$BENCHMARK/config.json")
+
+run_benchmark "$BENCHMARK" "$PATTERN" "$FIELD"
+
+echo "Benchmark completed successfully."

regression/benchmark/getpid/config.json

@@ -0,0 +1,5 @@
+{
+    "alert_threshold": "125%",
+    "pattern": "Syscall average latency:",
+    "field": "4"
+}

regression/benchmark/getpid/result_template.json

@@ -0,0 +1,14 @@
+[
+    {
+        "name": "Average Syscall Latency on Linux",
+        "unit": "ns",
+        "value": 0,
+        "extra": "linux_avg"
+    },
+    {
+        "name": "Average Syscall Latency on Asterinas",
+        "unit": "ns",
+        "value": 0,
+        "extra": "aster_avg"
+    }
+]

regression/benchmark/getpid/run.sh

@@ -0,0 +1,9 @@
+#!/bin/sh
+
+# SPDX-License-Identifier: MPL-2.0
+
+set -e
+
+echo "*** Running getpid ***"
+
+/benchmark/bin/getpid

regression/benchmark/sysbench-cpu/config.json

@@ -0,0 +1,5 @@
+{
+    "alert_threshold": "130%",
+    "pattern": "avg:",
+    "field": "NF"
+}

regression/benchmark/sysbench-cpu/result_template.json

@@ -0,0 +1,14 @@
+[
+    {
+        "name": "Average Execution Time per CPU on Linux",
+        "unit": "ms",
+        "value": 0,
+        "extra": "linux_avg"
+    },
+    {
+        "name": "Average Execution Time per CPU on Asterinas",
+        "unit": "ms",
+        "value": 0,
+        "extra": "aster_avg"
+    }
+]

regression/benchmark/sysbench-cpu/run.sh

@@ -0,0 +1,16 @@
+#!/bin/sh
+
+# SPDX-License-Identifier: MPL-2.0
+
+set -e
+
+TEST_TIME=${1:-60}
+TEST_THREADS=${2:-4}
+
+echo "*** Doing sysbench CPU test with ${TEST_THREADS} threads for ${TEST_TIME} seconds ***"
+
+/benchmark/bin/sysbench cpu \
+    --threads=${TEST_THREADS} \
+    --time=${TEST_TIME} \
+    --cpu-max-prime=20000 run
+    

regression/benchmark/sysbench-thread/config.json

@@ -0,0 +1,5 @@
+{
+    "alert_threshold": "130%",
+    "pattern": "avg:",
+    "field": "NF"
+}

regression/benchmark/sysbench-thread/result_template.json

@@ -0,0 +1,14 @@
+[
+    {
+        "name": "Average Execution Time per Thread on Linux",
+        "unit": "ms",
+        "value": 0,
+        "extra": "linux_avg"
+    },
+    {
+        "name": "Average Execution Time per Thread on Asterinas",
+        "unit": "ms",
+        "value": 0,
+        "extra": "aster_avg"
+    }
+]

regression/benchmark/sysbench-thread/run.sh

@@ -0,0 +1,16 @@
+#!/bin/sh
+
+# SPDX-License-Identifier: MPL-2.0
+
+set -e
+
+TEST_TIME=${1:-60}
+TEST_THREADS=${2:-200}
+
+echo "*** Doing sysbench with ${TEST_THREADS} threads for ${TEST_TIME} seconds ***"
+
+/benchmark/bin/sysbench threads \
+    --threads=${TEST_THREADS} \
+    --thread-yields=100 \
+    --thread-locks=4 \
+    --time=${TEST_TIME} run