diff --git a/.github/workflows/benchmark_asterinas.yml b/.github/workflows/benchmark_asterinas.yml index 85d7d7d06..bbd08c63e 100644 --- a/.github/workflows/benchmark_asterinas.yml +++ b/.github/workflows/benchmark_asterinas.yml @@ -61,6 +61,7 @@ jobs: - lmbench/tcp_loopback_select_lat - lmbench/tcp_loopback_http_bw - lmbench/udp_loopback_lat + - iperf3/tcp_virtio_bw fail-fast: false timeout-minutes: 60 container: @@ -89,7 +90,8 @@ jobs: max_attempts: 3 command: | make install_osdk - bash test/benchmark/bench_linux_and_aster.sh ${{ matrix.benchmark }} + BENCHMARK_TYPE=$(jq -r '.benchmark_type' test/benchmark/${{ matrix.benchmark }}/config.json) + bash test/benchmark/bench_linux_and_aster.sh ${{ matrix.benchmark }} $BENCHMARK_TYPE - name: Set up benchmark configuration run: | diff --git a/test/benchmark/README.md b/test/benchmark/README.md index 9f007b0a5..f92fc7b0c 100644 --- a/test/benchmark/README.md +++ b/test/benchmark/README.md @@ -83,7 +83,8 @@ To add a new benchmark to the Asternias Continuous Integration (CI) system, foll "search_pattern": "Simple syscall:", "result_index": "3", "description": "lat_syscall null", - "title": "[Process] The cost of getpid" + "title": "[Process] The cost of getpid", + "benchmark_type": "host_guest" } ``` @@ -93,6 +94,9 @@ To add a new benchmark to the Asternias Continuous Integration (CI) system, foll - `result_index`: Specify the index of the result in the extracted output. This field is aligned with `awk`'s action. - `description`: Provide a brief description of the benchmark. - `title`: Set the title of the benchmark. + - `benchmark_type`: This parameter defines the type of benchmark to be executed. The default value is `guest_only`. The available options include `guest_only`, and `host_guest`. + - `guest_only`: Use this option when the benchmark is intended solely for the guest environment. + - `host_guest`: Choose this option when the benchmark involves both the host and guest environments. When using this option, you will need to define your own `host.sh` and `bench_runner.sh` scripts to handle the host-side operations and benchmark execution. For example, if the benchmark output is "Syscall average latency: 1000 ns", the `search_pattern` is "Syscall average latency:", and the `result_index` is "4". `awk` will extract "1000" as the benchmark result. See the `awk` [manual](https://www.gnu.org/software/gawk/manual/gawk.html#Getting-Started) for more information. diff --git a/test/benchmark/bench_linux_and_aster.sh b/test/benchmark/bench_linux_and_aster.sh index 886e40dff..f5783f140 100755 --- a/test/benchmark/bench_linux_and_aster.sh +++ b/test/benchmark/bench_linux_and_aster.sh @@ -10,46 +10,9 @@ command -v jq >/dev/null 2>&1 || { echo >&2 "jq is not installed. Aborting."; ex # Script directory BENCHMARK_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)" -# Dependencies for Linux -LINUX_DEPENDENCIES_DIR="/opt/linux_binary_cache" -LINUX_KERNEL="${LINUX_DEPENDENCIES_DIR}/vmlinuz" -LINUX_KERNEL_VERSION="5.15.0-105-generic" -LINUX_MODULES_DIR="${BENCHMARK_DIR}/../build/initramfs/lib/modules/${LINUX_KERNEL_VERSION}/kernel" -# Atomic wget script -WGET_SCRIPT="${BENCHMARK_DIR}/../../tools/atomic_wget.sh" -# Prepare Linux kernel and modules -prepare_libs() { - # Download the Linux kernel and modules - mkdir -p "${LINUX_DEPENDENCIES_DIR}" - - if [ ! -f "${LINUX_KERNEL}" ]; then - echo "Downloading the Linux kernel image..." - ${WGET_SCRIPT} "${LINUX_KERNEL}" "https://raw.githubusercontent.com/asterinas/linux_binary_cache/8a5b6fd/vmlinuz-${LINUX_KERNEL_VERSION}" || { - echo "Failed to download the Linux kernel image." - exit 1 - } - fi - if [ ! -f "${LINUX_DEPENDENCIES_DIR}/virtio_blk.ko" ]; then - echo "Downloading the virtio_blk kernel module..." - ${WGET_SCRIPT} "${LINUX_DEPENDENCIES_DIR}/virtio_blk.ko" "https://raw.githubusercontent.com/asterinas/linux_binary_cache/8a5b6fd/kernel/drivers/block/virtio_blk.ko" || { - echo "Failed to download the Linux kernel module." - exit 1 - } - fi - # Copy the kernel modules to the initramfs directory - if [ ! -f "${LINUX_MODULES_DIR}/drivers/block/virtio_blk.ko" ]; then - mkdir -p "${LINUX_MODULES_DIR}/drivers/block" - cp ${LINUX_DEPENDENCIES_DIR}/virtio_blk.ko "${LINUX_MODULES_DIR}/drivers/block/virtio_blk.ko" - fi -} - -# Prepare fs for Linux -prepare_fs() { - # Disable unsupported ext2 features of Asterinas on Linux to ensure fairness - mke2fs -F -O ^ext_attr -O ^resize_inode -O ^dir_index ${BENCHMARK_DIR}/../build/ext2.img - make initramfs -} +# Source the prepare_host.sh script +source "${BENCHMARK_DIR}/common/prepare_host.sh" # Parse the results from the benchmark output parse_results() { @@ -80,23 +43,21 @@ parse_results() { # Run the benchmark on Linux and Asterinas run_benchmark() { local benchmark="$1" - local search_pattern="$2" - local result_index="$3" + local benchmark_type="$2" + local search_pattern="$3" + local result_index="$4" local linux_output="${BENCHMARK_DIR}/linux_output.txt" local aster_output="${BENCHMARK_DIR}/aster_output.txt" local result_template="${BENCHMARK_DIR}/${benchmark}/result_template.json" local benchmark_name=$(basename "${benchmark}") + local benchmark_root=$(dirname "${benchmark}") local result_file="result_${benchmark_name}.json" echo "Preparing libraries..." prepare_libs - local asterinas_cmd="make run BENCHMARK=${benchmark} ENABLE_KVM=1 RELEASE_LTO=1 2>&1 | tee ${aster_output}" - echo "Running benchmark ${benchmark} on Asterinas..." - eval "$asterinas_cmd" - - prepare_fs + local asterinas_cmd="make run BENCHMARK=${benchmark} ENABLE_KVM=1 RELEASE_LTO=1 2>&1" local linux_cmd="/usr/local/qemu/bin/qemu-system-x86_64 \ --no-reboot \ -smp 1 \ @@ -109,10 +70,37 @@ run_benchmark() { -drive if=none,format=raw,id=x0,file=${BENCHMARK_DIR}/../build/ext2.img \ -device virtio-blk-pci,bus=pcie.0,addr=0x6,drive=x0,serial=vext2,disable-legacy=on,disable-modern=off,queue-size=64,num-queues=1,config-wce=off,request-merging=off,write-cache=off,backend_defaults=off,discard=off,event_idx=off,indirect_desc=off,ioeventfd=off,queue_reset=off \ -append 'console=ttyS0 rdinit=/benchmark/common/bench_runner.sh ${benchmark} linux mitigations=off hugepages=0 transparent_hugepage=never' \ + -netdev user,id=net01,hostfwd=tcp::5201-:5201 \ + -device virtio-net-pci,netdev=net01,disable-legacy=on,disable-modern=off \ -nographic \ - 2>&1 | tee ${linux_output}" - echo "Running benchmark ${benchmark} on Linux..." - eval "$linux_cmd" + 2>&1" + case "${benchmark_type}" in + "guest_only") + echo "Running benchmark ${benchmark} on Asterinas..." + eval "$asterinas_cmd" | tee ${aster_output} + prepare_fs + echo "Running benchmark ${benchmark} on Linux..." + eval "$linux_cmd" | tee ${linux_output} + ;; + "host_guest") + echo "Running benchmark ${benchmark} on host and guest..." + bash "${BENCHMARK_DIR}/${benchmark_root}/bench_runner.sh" \ + "${BENCHMARK_DIR}/${benchmark}" \ + "${asterinas_cmd}" \ + "${linux_cmd}" \ + "${aster_output}" \ + "${linux_output}" + ;; + "guest-guest") + echo "Running benchmark ${benchmark} between guests..." + echo "TODO" + exit 1 + ;; + *) + echo "Error: Unknown benchmark type '${benchmark_type}'" >&2 + exit 1 + ;; + esac echo "Parsing results..." parse_results "$benchmark" "$search_pattern" "$result_index" "$linux_output" "$aster_output" "$result_template" "$result_file" @@ -125,6 +113,11 @@ run_benchmark() { # Main BENCHMARK="$1" +if [ -z "$2" ] || [ "$2" = "null" ]; then + BENCHMARK_TYPE="guest_only" +else + BENCHMARK_TYPE="$2" +fi echo "Running benchmark ${BENCHMARK}..." pwd @@ -136,6 +129,6 @@ fi search_pattern=$(jq -r '.search_pattern' "$BENCHMARK_DIR/$BENCHMARK/config.json") result_index=$(jq -r '.result_index' "$BENCHMARK_DIR/$BENCHMARK/config.json") -run_benchmark "$BENCHMARK" "$search_pattern" "$result_index" +run_benchmark "$BENCHMARK" "$BENCHMARK_TYPE" "$search_pattern" "$result_index" echo "Benchmark completed successfully." diff --git a/test/benchmark/common/bench_runner.sh b/test/benchmark/common/bench_runner.sh index d98ba01ee..05b3961f8 100755 --- a/test/benchmark/common/bench_runner.sh +++ b/test/benchmark/common/bench_runner.sh @@ -8,7 +8,8 @@ set -e BENCHMARK_DIR="/benchmark" BENCH_NAME=$1 -SYSTEM=$2 +SYSTEM="${2:-asterinas}" +echo "Running benchmark: ${BENCH_NAME} on ${SYSTEM}" print_help() { echo "Usage: $0 " @@ -42,9 +43,19 @@ prepare_system() { # System-specific preparation if [ "$SYSTEM" = "linux" ]; then + # Mount necessary fs mount -t devtmpfs devtmpfs /dev - ip link set lo up + # Add drivers + depmod + modprobe failover + modprobe net_failover + modprobe virtio_net modprobe virtio_blk + # Enable network + ip link set lo up + ip link set eth0 up + ifconfig eth0 10.0.2.15 + # Mount ext2 mount -t ext2 /dev/vda /ext2 elif [ "$SYSTEM" = "asterinas" ]; then # Asterinas-specific preparation (if any) diff --git a/test/benchmark/common/prepare_host.sh b/test/benchmark/common/prepare_host.sh new file mode 100644 index 000000000..eab18f05c --- /dev/null +++ b/test/benchmark/common/prepare_host.sh @@ -0,0 +1,66 @@ +#!/bin/bash + +# SPDX-License-Identifier: MPL-2.0 + +set -e +set -o pipefail + +# Set BENCHMARK_DIR to the parent directory of the current directory if it is not set +BENCHMARK_DIR="${BENCHMARK_DIR:-$(cd "$(dirname "${BASH_SOURCE[0]}")/.." &>/dev/null && pwd)}" +# Dependencies for Linux +LINUX_DEPENDENCIES_DIR="/opt/linux_binary_cache" +LINUX_KERNEL="${LINUX_DEPENDENCIES_DIR}/vmlinuz" +LINUX_KERNEL_VERSION="5.15.0-105-generic" +LINUX_MODULES_DIR="${BENCHMARK_DIR}/../build/initramfs/lib/modules/${LINUX_KERNEL_VERSION}/kernel" +WGET_SCRIPT="${BENCHMARK_DIR}/../../tools/atomic_wget.sh" + +# Prepare Linux kernel and modules +prepare_libs() { + # Download the Linux kernel and modules + mkdir -p "${LINUX_DEPENDENCIES_DIR}" + + # Array of files to download and their URLs + declare -A files=( + ["${LINUX_KERNEL}"]="https://raw.githubusercontent.com/asterinas/linux_binary_cache/8a5b6fd/vmlinuz-${LINUX_KERNEL_VERSION}" + ["${LINUX_DEPENDENCIES_DIR}/virtio_blk.ko"]="https://raw.githubusercontent.com/asterinas/linux_binary_cache/8a5b6fd/kernel/drivers/block/virtio_blk.ko" + ["${LINUX_DEPENDENCIES_DIR}/virtio_net.ko"]="https://raw.githubusercontent.com/asterinas/linux_binary_cache/8a5b6fd/kernel/drivers/net/virtio_net.ko" + ["${LINUX_DEPENDENCIES_DIR}/net_failover.ko"]="https://raw.githubusercontent.com/asterinas/linux_binary_cache/8a5b6fd/kernel/drivers/net/net_failover.ko" + ["${LINUX_DEPENDENCIES_DIR}/failover.ko"]="https://raw.githubusercontent.com/asterinas/linux_binary_cache/8a5b6fd/kernel/net/core/failover.ko" + ) + + # Download files if they don't exist + for file in "${!files[@]}"; do + if [ ! -f "$file" ]; then + echo "Downloading ${file##*/}..." + ${WGET_SCRIPT} "$file" "${files[$file]}" || { + echo "Failed to download ${file##*/}." + exit 1 + } + fi + done + + # Copy the kernel modules to the initramfs directory + if [ ! -f "${LINUX_MODULES_DIR}/drivers/block/virtio_blk.ko" ]; then + mkdir -p "${LINUX_MODULES_DIR}/drivers/block" + mkdir -p "${LINUX_MODULES_DIR}/drivers/net" + mkdir -p "${LINUX_MODULES_DIR}/net/core" + + declare -A modules=( + ["${LINUX_DEPENDENCIES_DIR}/virtio_blk.ko"]="${LINUX_MODULES_DIR}/drivers/block/virtio_blk.ko" + ["${LINUX_DEPENDENCIES_DIR}/virtio_net.ko"]="${LINUX_MODULES_DIR}/drivers/net/virtio_net.ko" + ["${LINUX_DEPENDENCIES_DIR}/net_failover.ko"]="${LINUX_MODULES_DIR}/drivers/net/net_failover.ko" + ["${LINUX_DEPENDENCIES_DIR}/failover.ko"]="${LINUX_MODULES_DIR}/net/core/failover.ko" + ) + + for src in "${!modules[@]}"; do + sudo cp "$src" "${modules[$src]}" + done + fi +} + +# Prepare fs for Linux +prepare_fs() { + # Disable unsupported ext2 features of Asterinas on Linux to ensure fairness + mke2fs -F -O ^ext_attr -O ^resize_inode -O ^dir_index ${BENCHMARK_DIR}/../build/ext2.img + make initramfs +} \ No newline at end of file diff --git a/test/benchmark/iperf3/bench_runner.sh b/test/benchmark/iperf3/bench_runner.sh new file mode 100755 index 000000000..f5754cd1b --- /dev/null +++ b/test/benchmark/iperf3/bench_runner.sh @@ -0,0 +1,46 @@ +#!/bin/bash + +# SPDX-License-Identifier: MPL-2.0 + +set -e + +BENCHMARK_PATH=$1 +ASTERINAS_GUEST_CMD=$2 +LINUX_GUEST_CMD=$3 +ASTERINAS_OUTPUT=$4 +LINUX_OUTPUT=$5 + +# Import the common functions +BENCHMARK_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)/../" +source "${BENCHMARK_DIR}/common/prepare_host.sh" + +# Persist iperf3 port +export IPERF_PORT=5201 + +# Function to run the benchmark +# Parameters: +# $1: guest command to run on the VM +# $2: output file to store the benchmark results +# $3: sleep time before running the host command +run_benchmark() { + local guest_cmd=$1 + local output_file=$2 + local sleep_time=$3 + + echo "Running the benchmark on the VM..." + eval "${guest_cmd}" & + sleep "${sleep_time}" + # Run the host command and save the output to the specified file. + # You can also redirect the guest output to it. + echo "Running the benchmark on the host..." + bash "${BENCHMARK_PATH}/host.sh" | tee "${output_file}" +} + +# Run the benchmark on the Asterinas VM +# Use a sleep time of 2 minutes (2m) for the Asterinas VM +run_benchmark "${ASTERINAS_GUEST_CMD}" "${ASTERINAS_OUTPUT}" "2m" + +# Run the benchmark on the Linux VM +# Use a sleep time of 20 seconds (20s) for the Linux VM +prepare_fs +run_benchmark "${LINUX_GUEST_CMD}" "${LINUX_OUTPUT}" "20s" \ No newline at end of file diff --git a/test/benchmark/iperf3/summary.json b/test/benchmark/iperf3/summary.json new file mode 100644 index 000000000..c0a261392 --- /dev/null +++ b/test/benchmark/iperf3/summary.json @@ -0,0 +1,5 @@ +{ + "benchmarks": [ + "tcp_virtio_bw" + ] +} \ No newline at end of file diff --git a/test/benchmark/iperf3/tcp_virtio_bw/config.json b/test/benchmark/iperf3/tcp_virtio_bw/config.json new file mode 100644 index 000000000..8ec792e8c --- /dev/null +++ b/test/benchmark/iperf3/tcp_virtio_bw/config.json @@ -0,0 +1,9 @@ +{ + "alert_threshold": "130%", + "alert_tool": "customBiggerIsBetter", + "search_pattern": "sender", + "result_index": "7", + "description": "iperf3 -s -B 10.0.2.15", + "title": "[Network] iperf3 sender performance using TCP", + "benchmark_type": "host_guest" +} \ No newline at end of file diff --git a/test/benchmark/iperf3/tcp_virtio_bw/host.sh b/test/benchmark/iperf3/tcp_virtio_bw/host.sh new file mode 100755 index 000000000..cb7439d8f --- /dev/null +++ b/test/benchmark/iperf3/tcp_virtio_bw/host.sh @@ -0,0 +1,19 @@ +#!/bin/sh + +# SPDX-License-Identifier: MPL-2.0 + +set -e + +# Function to stop the guest VM +stop_guest() { + echo "Stopping guest VM..." + pgrep qemu | xargs kill +} + +# Trap EXIT signal to ensure guest VM is stopped on script exit +trap stop_guest EXIT + +# Run iperf3 client +/usr/local/benchmark/iperf/bin/iperf3 -c 127.0.0.1 -f m + +# The trap will automatically stop the guest VM when the script exits \ No newline at end of file diff --git a/test/benchmark/iperf3/tcp_virtio_bw/result_template.json b/test/benchmark/iperf3/tcp_virtio_bw/result_template.json new file mode 100644 index 000000000..000c78bce --- /dev/null +++ b/test/benchmark/iperf3/tcp_virtio_bw/result_template.json @@ -0,0 +1,14 @@ +[ + { + "name": "Average TCP Bandwidth over virtio-net between Host Linux and Guest Linux", + "unit": "Mbits/sec", + "value": 0, + "extra": "linux_result" + }, + { + "name": "Average TCP Bandwidth over virtio-net between Host Linux and Guest Asterinas", + "unit": "Mbits/sec", + "value": 0, + "extra": "aster_result" + } +] \ No newline at end of file diff --git a/test/benchmark/iperf3/tcp_virtio_bw/run.sh b/test/benchmark/iperf3/tcp_virtio_bw/run.sh new file mode 100644 index 000000000..857f21500 --- /dev/null +++ b/test/benchmark/iperf3/tcp_virtio_bw/run.sh @@ -0,0 +1,8 @@ +#!/bin/sh + +# SPDX-License-Identifier: MPL-2.0 + +set -e + +echo "Running iperf3 server..." +/benchmark/bin/iperf3 -s -B 10.0.2.15 --one-off \ No newline at end of file diff --git a/tools/qemu_args.sh b/tools/qemu_args.sh index 612201411..99d0b644d 100755 --- a/tools/qemu_args.sh +++ b/tools/qemu_args.sh @@ -7,11 +7,12 @@ # A switch "-ovmf" can be passed as an argument to enable OVMF. # The enrivonmental variable VSOCK can be passed as 1 to trigger vsock module. -SSH_RAND_PORT=$(shuf -i 1024-65535 -n 1) -NGINX_RAND_PORT=$(shuf -i 1024-65535 -n 1) -REDIS_RAND_PORT=$(shuf -i 1024-65535 -n 1) +SSH_RAND_PORT=${SSH_PORT:-$(shuf -i 1024-65535 -n 1)} +NGINX_RAND_PORT=${NGINX_PORT:-$(shuf -i 1024-65535 -n 1)} +REDIS_RAND_PORT=${REDIS_PORT:-$(shuf -i 1024-65535 -n 1)} +IPERF_RAND_PORT=${IPERF_PORT:-$(shuf -i 1024-65535 -n 1)} -echo "[$1] Forwarded QEMU guest port: $SSH_RAND_PORT->22; $NGINX_RAND_PORT->8080 $REDIS_RAND_PORT->6379" 1>&2 +echo "[$1] Forwarded QEMU guest port: $SSH_RAND_PORT->22; $NGINX_RAND_PORT->8080 $REDIS_RAND_PORT->6379 $IPERF_RAND_PORT->5201" 1>&2 COMMON_QEMU_ARGS="\ -cpu Icelake-Server,+x2apic \ @@ -23,7 +24,7 @@ COMMON_QEMU_ARGS="\ -serial chardev:mux \ -monitor chardev:mux \ -chardev stdio,id=mux,mux=on,signal=off,logfile=qemu.log \ - -netdev user,id=net01,hostfwd=tcp::$SSH_RAND_PORT-:22,hostfwd=tcp::$NGINX_RAND_PORT-:8080,hostfwd=tcp::$REDIS_RAND_PORT-:6379 \ + -netdev user,id=net01,hostfwd=tcp::$SSH_RAND_PORT-:22,hostfwd=tcp::$NGINX_RAND_PORT-:8080,hostfwd=tcp::$REDIS_RAND_PORT-:6379,hostfwd=tcp::$IPERF_RAND_PORT-:5201 \ -object filter-dump,id=filter0,netdev=net01,file=virtio-net.pcap \ -device isa-debug-exit,iobase=0xf4,iosize=0x04 \ -drive if=none,format=raw,id=x0,file=./test/build/ext2.img \