Add iperf3 benchmark

.github/workflows/benchmark_asterinas.yml

@@ -61,6 +61,7 @@ jobs:
           - lmbench/tcp_loopback_select_lat
           - lmbench/tcp_loopback_http_bw
           - lmbench/udp_loopback_lat
+          - iperf3/tcp_virtio_bw
       fail-fast: false
     timeout-minutes: 60
     container: 
@@ -89,7 +90,8 @@ jobs:
         max_attempts: 3
         command: |
           make install_osdk
-          bash test/benchmark/bench_linux_and_aster.sh ${{ matrix.benchmark }}
+          BENCHMARK_TYPE=$(jq -r '.benchmark_type' test/benchmark/${{ matrix.benchmark }}/config.json)
+          bash test/benchmark/bench_linux_and_aster.sh ${{ matrix.benchmark }} $BENCHMARK_TYPE
 
     - name: Set up benchmark configuration
       run: |

test/benchmark/README.md

@@ -83,7 +83,8 @@ To add a new benchmark to the Asternias Continuous Integration (CI) system, foll
         "search_pattern": "Simple syscall:",
         "result_index": "3",
         "description": "lat_syscall null",
-        "title": "[Process] The cost of getpid"
+        "title": "[Process] The cost of getpid",
+        "benchmark_type": "host_guest"
       } 
      ```
      
@@ -93,6 +94,9 @@ To add a new benchmark to the Asternias Continuous Integration (CI) system, foll
     - `result_index`: Specify the index of the result in the extracted output. This field is aligned with `awk`'s action.
     - `description`: Provide a brief description of the benchmark.
     - `title`: Set the title of the benchmark.
+    - `benchmark_type`: This parameter defines the type of benchmark to be executed. The default value is `guest_only`. The available options include `guest_only`, and `host_guest`.
+      - `guest_only`: Use this option when the benchmark is intended solely for the guest environment.
+      - `host_guest`: Choose this option when the benchmark involves both the host and guest environments. When using this option, you will need to define your own `host.sh` and `bench_runner.sh` scripts to handle the host-side operations and benchmark execution.
 
     For example, if the benchmark output is "Syscall average latency: 1000 ns", the `search_pattern` is "Syscall average latency:", and the `result_index` is "4". `awk` will extract "1000" as the benchmark result. See the `awk` [manual](https://www.gnu.org/software/gawk/manual/gawk.html#Getting-Started) for more information.
 

test/benchmark/bench_linux_and_aster.sh

@@ -10,46 +10,9 @@ command -v jq >/dev/null 2>&1 || { echo >&2 "jq is not installed. Aborting."; ex
 
 # Script directory
 BENCHMARK_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)"
-# Dependencies for Linux
-LINUX_DEPENDENCIES_DIR="/opt/linux_binary_cache"
-LINUX_KERNEL="${LINUX_DEPENDENCIES_DIR}/vmlinuz"
-LINUX_KERNEL_VERSION="5.15.0-105-generic"
-LINUX_MODULES_DIR="${BENCHMARK_DIR}/../build/initramfs/lib/modules/${LINUX_KERNEL_VERSION}/kernel"
-# Atomic wget script
-WGET_SCRIPT="${BENCHMARK_DIR}/../../tools/atomic_wget.sh"
 
-# Prepare Linux kernel and modules
+# Source the prepare_host.sh script
-prepare_libs() {
+source "${BENCHMARK_DIR}/common/prepare_host.sh"
-    # Download the Linux kernel and modules
-    mkdir -p "${LINUX_DEPENDENCIES_DIR}"
-
-    if [ ! -f "${LINUX_KERNEL}" ]; then
-        echo "Downloading the Linux kernel image..."
-        ${WGET_SCRIPT} "${LINUX_KERNEL}" "https://raw.githubusercontent.com/asterinas/linux_binary_cache/8a5b6fd/vmlinuz-${LINUX_KERNEL_VERSION}" || {
-            echo "Failed to download the Linux kernel image."
-            exit 1
-        }
-    fi
-    if [ ! -f "${LINUX_DEPENDENCIES_DIR}/virtio_blk.ko" ]; then
-        echo "Downloading the virtio_blk kernel module..."
-        ${WGET_SCRIPT} "${LINUX_DEPENDENCIES_DIR}/virtio_blk.ko" "https://raw.githubusercontent.com/asterinas/linux_binary_cache/8a5b6fd/kernel/drivers/block/virtio_blk.ko" || {
-            echo "Failed to download the Linux kernel module."
-            exit 1
-        }
-    fi
-    # Copy the kernel modules to the initramfs directory
-    if [ ! -f "${LINUX_MODULES_DIR}/drivers/block/virtio_blk.ko" ]; then
-        mkdir -p "${LINUX_MODULES_DIR}/drivers/block"
-        cp ${LINUX_DEPENDENCIES_DIR}/virtio_blk.ko "${LINUX_MODULES_DIR}/drivers/block/virtio_blk.ko" 
-    fi
-}
-
-# Prepare fs for Linux
-prepare_fs() {
-    # Disable unsupported ext2 features of Asterinas on Linux to ensure fairness
-    mke2fs -F -O ^ext_attr -O ^resize_inode -O ^dir_index ${BENCHMARK_DIR}/../build/ext2.img
-    make initramfs
-}
 
 # Parse the results from the benchmark output
 parse_results() {
@@ -80,23 +43,21 @@ parse_results() {
 # Run the benchmark on Linux and Asterinas
 run_benchmark() {
     local benchmark="$1"
-    local search_pattern="$2"
+    local benchmark_type="$2"
-    local result_index="$3"
+    local search_pattern="$3"
+    local result_index="$4"
 
     local linux_output="${BENCHMARK_DIR}/linux_output.txt"
     local aster_output="${BENCHMARK_DIR}/aster_output.txt"
     local result_template="${BENCHMARK_DIR}/${benchmark}/result_template.json"
     local benchmark_name=$(basename "${benchmark}")
+    local benchmark_root=$(dirname "${benchmark}")
     local result_file="result_${benchmark_name}.json"
     
     echo "Preparing libraries..."
     prepare_libs
 
-    local asterinas_cmd="make run BENCHMARK=${benchmark} ENABLE_KVM=1 RELEASE_LTO=1 2>&1 | tee ${aster_output}"
+    local asterinas_cmd="make run BENCHMARK=${benchmark} ENABLE_KVM=1 RELEASE_LTO=1 2>&1"
-    echo "Running benchmark ${benchmark} on Asterinas..."
-    eval "$asterinas_cmd"
-
-    prepare_fs
     local linux_cmd="/usr/local/qemu/bin/qemu-system-x86_64 \
         --no-reboot \
         -smp 1 \
@@ -109,10 +70,37 @@ run_benchmark() {
         -drive if=none,format=raw,id=x0,file=${BENCHMARK_DIR}/../build/ext2.img \
         -device virtio-blk-pci,bus=pcie.0,addr=0x6,drive=x0,serial=vext2,disable-legacy=on,disable-modern=off,queue-size=64,num-queues=1,config-wce=off,request-merging=off,write-cache=off,backend_defaults=off,discard=off,event_idx=off,indirect_desc=off,ioeventfd=off,queue_reset=off \
         -append 'console=ttyS0 rdinit=/benchmark/common/bench_runner.sh ${benchmark} linux mitigations=off hugepages=0 transparent_hugepage=never' \
+        -netdev user,id=net01,hostfwd=tcp::5201-:5201 \
+        -device virtio-net-pci,netdev=net01,disable-legacy=on,disable-modern=off \
         -nographic \
-        2>&1 | tee ${linux_output}"
+        2>&1"
-    echo "Running benchmark ${benchmark} on Linux..."
+    case "${benchmark_type}" in
-    eval "$linux_cmd"
+        "guest_only")
+            echo "Running benchmark ${benchmark} on Asterinas..."
+            eval "$asterinas_cmd" | tee ${aster_output}
+            prepare_fs
+            echo "Running benchmark ${benchmark} on Linux..."
+            eval "$linux_cmd" | tee ${linux_output}
+            ;;
+        "host_guest")
+            echo "Running benchmark ${benchmark} on host and guest..."
+            bash "${BENCHMARK_DIR}/${benchmark_root}/bench_runner.sh" \
+                "${BENCHMARK_DIR}/${benchmark}" \
+                "${asterinas_cmd}" \
+                "${linux_cmd}" \
+                "${aster_output}" \
+                "${linux_output}"
+            ;;
+        "guest-guest")
+            echo "Running benchmark ${benchmark} between guests..."
+            echo "TODO"
+            exit 1
+            ;;
+        *)
+            echo "Error: Unknown benchmark type '${benchmark_type}'" >&2
+            exit 1
+            ;;
+    esac
 
     echo "Parsing results..."
     parse_results "$benchmark" "$search_pattern" "$result_index" "$linux_output" "$aster_output" "$result_template" "$result_file"
@@ -125,6 +113,11 @@ run_benchmark() {
 # Main
 
 BENCHMARK="$1"
+if [ -z "$2" ] || [ "$2" = "null" ]; then
+    BENCHMARK_TYPE="guest_only"
+else
+    BENCHMARK_TYPE="$2"
+fi
 
 echo "Running benchmark ${BENCHMARK}..."
 pwd
@@ -136,6 +129,6 @@ fi
 search_pattern=$(jq -r '.search_pattern' "$BENCHMARK_DIR/$BENCHMARK/config.json")
 result_index=$(jq -r '.result_index' "$BENCHMARK_DIR/$BENCHMARK/config.json")
 
-run_benchmark "$BENCHMARK" "$search_pattern" "$result_index"
+run_benchmark "$BENCHMARK" "$BENCHMARK_TYPE" "$search_pattern" "$result_index"
 
 echo "Benchmark completed successfully."

test/benchmark/common/bench_runner.sh

@@ -8,7 +8,8 @@ set -e
 BENCHMARK_DIR="/benchmark"
 
 BENCH_NAME=$1
-SYSTEM=$2
+SYSTEM="${2:-asterinas}"
+echo "Running benchmark: ${BENCH_NAME} on ${SYSTEM}"
 
 print_help() {
     echo "Usage: $0 <benchmark_name> <system_type>"
@@ -42,9 +43,19 @@ prepare_system() {
     
     # System-specific preparation
     if [ "$SYSTEM" = "linux" ]; then
+        # Mount necessary fs
         mount -t devtmpfs devtmpfs /dev
-        ip link set lo up
+        # Add drivers
+        depmod
+        modprobe failover
+        modprobe net_failover
+        modprobe virtio_net
         modprobe virtio_blk
+        # Enable network
+        ip link set lo up
+        ip link set eth0 up
+        ifconfig eth0 10.0.2.15
+        # Mount ext2
         mount -t ext2 /dev/vda /ext2
     elif [ "$SYSTEM" = "asterinas" ]; then
         # Asterinas-specific preparation (if any)

test/benchmark/common/prepare_host.sh

@@ -0,0 +1,66 @@
+#!/bin/bash
+
+# SPDX-License-Identifier: MPL-2.0
+
+set -e
+set -o pipefail
+
+# Set BENCHMARK_DIR to the parent directory of the current directory if it is not set
+BENCHMARK_DIR="${BENCHMARK_DIR:-$(cd "$(dirname "${BASH_SOURCE[0]}")/.." &>/dev/null && pwd)}"
+# Dependencies for Linux
+LINUX_DEPENDENCIES_DIR="/opt/linux_binary_cache"
+LINUX_KERNEL="${LINUX_DEPENDENCIES_DIR}/vmlinuz"
+LINUX_KERNEL_VERSION="5.15.0-105-generic"
+LINUX_MODULES_DIR="${BENCHMARK_DIR}/../build/initramfs/lib/modules/${LINUX_KERNEL_VERSION}/kernel"
+WGET_SCRIPT="${BENCHMARK_DIR}/../../tools/atomic_wget.sh"
+
+# Prepare Linux kernel and modules
+prepare_libs() {
+    # Download the Linux kernel and modules
+    mkdir -p "${LINUX_DEPENDENCIES_DIR}"
+
+    # Array of files to download and their URLs
+    declare -A files=(
+        ["${LINUX_KERNEL}"]="https://raw.githubusercontent.com/asterinas/linux_binary_cache/8a5b6fd/vmlinuz-${LINUX_KERNEL_VERSION}"
+        ["${LINUX_DEPENDENCIES_DIR}/virtio_blk.ko"]="https://raw.githubusercontent.com/asterinas/linux_binary_cache/8a5b6fd/kernel/drivers/block/virtio_blk.ko"
+        ["${LINUX_DEPENDENCIES_DIR}/virtio_net.ko"]="https://raw.githubusercontent.com/asterinas/linux_binary_cache/8a5b6fd/kernel/drivers/net/virtio_net.ko"
+        ["${LINUX_DEPENDENCIES_DIR}/net_failover.ko"]="https://raw.githubusercontent.com/asterinas/linux_binary_cache/8a5b6fd/kernel/drivers/net/net_failover.ko"
+        ["${LINUX_DEPENDENCIES_DIR}/failover.ko"]="https://raw.githubusercontent.com/asterinas/linux_binary_cache/8a5b6fd/kernel/net/core/failover.ko"
+    )
+
+    # Download files if they don't exist
+    for file in "${!files[@]}"; do
+        if [ ! -f "$file" ]; then
+            echo "Downloading ${file##*/}..."
+            ${WGET_SCRIPT} "$file" "${files[$file]}" || {
+                echo "Failed to download ${file##*/}."
+                exit 1
+            }
+        fi
+    done
+
+    # Copy the kernel modules to the initramfs directory
+    if [ ! -f "${LINUX_MODULES_DIR}/drivers/block/virtio_blk.ko" ]; then
+        mkdir -p "${LINUX_MODULES_DIR}/drivers/block"
+        mkdir -p "${LINUX_MODULES_DIR}/drivers/net"
+        mkdir -p "${LINUX_MODULES_DIR}/net/core"
+
+        declare -A modules=(
+            ["${LINUX_DEPENDENCIES_DIR}/virtio_blk.ko"]="${LINUX_MODULES_DIR}/drivers/block/virtio_blk.ko"
+            ["${LINUX_DEPENDENCIES_DIR}/virtio_net.ko"]="${LINUX_MODULES_DIR}/drivers/net/virtio_net.ko"
+            ["${LINUX_DEPENDENCIES_DIR}/net_failover.ko"]="${LINUX_MODULES_DIR}/drivers/net/net_failover.ko"
+            ["${LINUX_DEPENDENCIES_DIR}/failover.ko"]="${LINUX_MODULES_DIR}/net/core/failover.ko"
+        )
+
+        for src in "${!modules[@]}"; do
+            sudo cp "$src" "${modules[$src]}"
+        done
+    fi
+}
+
+# Prepare fs for Linux
+prepare_fs() {
+    # Disable unsupported ext2 features of Asterinas on Linux to ensure fairness
+    mke2fs -F -O ^ext_attr -O ^resize_inode -O ^dir_index ${BENCHMARK_DIR}/../build/ext2.img
+    make initramfs
+}

test/benchmark/iperf3/bench_runner.sh

@@ -0,0 +1,46 @@
+#!/bin/bash
+
+# SPDX-License-Identifier: MPL-2.0
+
+set -e
+
+BENCHMARK_PATH=$1
+ASTERINAS_GUEST_CMD=$2
+LINUX_GUEST_CMD=$3
+ASTERINAS_OUTPUT=$4
+LINUX_OUTPUT=$5
+
+# Import the common functions
+BENCHMARK_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)/../"
+source "${BENCHMARK_DIR}/common/prepare_host.sh"
+
+# Persist iperf3 port
+export IPERF_PORT=5201
+
+# Function to run the benchmark
+# Parameters:
+#   $1: guest command to run on the VM
+#   $2: output file to store the benchmark results
+#   $3: sleep time before running the host command
+run_benchmark() {
+    local guest_cmd=$1
+    local output_file=$2
+    local sleep_time=$3
+
+    echo "Running the benchmark on the VM..."
+    eval "${guest_cmd}" & 
+    sleep "${sleep_time}"  
+    # Run the host command and save the output to the specified file.
+    # You can also redirect the guest output to it.
+    echo "Running the benchmark on the host..."
+    bash "${BENCHMARK_PATH}/host.sh" | tee "${output_file}"  
+}
+
+# Run the benchmark on the Asterinas VM
+# Use a sleep time of 2 minutes (2m) for the Asterinas VM
+run_benchmark "${ASTERINAS_GUEST_CMD}" "${ASTERINAS_OUTPUT}" "2m"
+
+# Run the benchmark on the Linux VM
+# Use a sleep time of 20 seconds (20s) for the Linux VM
+prepare_fs
+run_benchmark "${LINUX_GUEST_CMD}" "${LINUX_OUTPUT}" "20s"

test/benchmark/iperf3/summary.json

@@ -0,0 +1,5 @@
+{
+    "benchmarks": [
+        "tcp_virtio_bw"
+    ]
+}

test/benchmark/iperf3/tcp_virtio_bw/config.json

@@ -0,0 +1,9 @@
+{
+    "alert_threshold": "130%",
+    "alert_tool": "customBiggerIsBetter",
+    "search_pattern": "sender",
+    "result_index": "7",
+    "description": "iperf3 -s -B 10.0.2.15",
+    "title": "[Network] iperf3 sender performance using TCP",
+    "benchmark_type": "host_guest"
+}

test/benchmark/iperf3/tcp_virtio_bw/host.sh

@@ -0,0 +1,19 @@
+#!/bin/sh
+
+# SPDX-License-Identifier: MPL-2.0
+
+set -e
+
+# Function to stop the guest VM
+stop_guest() {
+    echo "Stopping guest VM..."
+    pgrep qemu | xargs kill
+}
+
+# Trap EXIT signal to ensure guest VM is stopped on script exit
+trap stop_guest EXIT
+
+# Run iperf3 client
+/usr/local/benchmark/iperf/bin/iperf3 -c 127.0.0.1 -f m
+
+# The trap will automatically stop the guest VM when the script exits

test/benchmark/iperf3/tcp_virtio_bw/result_template.json

@@ -0,0 +1,14 @@
+[
+    {
+        "name": "Average TCP Bandwidth over virtio-net between Host Linux and Guest Linux",
+        "unit": "Mbits/sec",
+        "value": 0,
+        "extra": "linux_result"
+    },
+    {
+        "name": "Average TCP Bandwidth over virtio-net between Host Linux and Guest Asterinas",
+        "unit": "Mbits/sec",
+        "value": 0,
+        "extra": "aster_result"
+    }
+]

test/benchmark/iperf3/tcp_virtio_bw/run.sh

@@ -0,0 +1,8 @@
+#!/bin/sh
+
+# SPDX-License-Identifier: MPL-2.0
+
+set -e
+
+echo "Running iperf3 server..."
+/benchmark/bin/iperf3 -s -B 10.0.2.15 --one-off

tools/qemu_args.sh

@@ -7,11 +7,12 @@
 # A switch "-ovmf" can be passed as an argument to enable OVMF.
 # The enrivonmental variable VSOCK can be passed as 1 to trigger vsock module.
 
-SSH_RAND_PORT=$(shuf -i 1024-65535 -n 1)
+SSH_RAND_PORT=${SSH_PORT:-$(shuf -i 1024-65535 -n 1)}
-NGINX_RAND_PORT=$(shuf -i 1024-65535 -n 1)
+NGINX_RAND_PORT=${NGINX_PORT:-$(shuf -i 1024-65535 -n 1)}
-REDIS_RAND_PORT=$(shuf -i 1024-65535 -n 1)
+REDIS_RAND_PORT=${REDIS_PORT:-$(shuf -i 1024-65535 -n 1)}
+IPERF_RAND_PORT=${IPERF_PORT:-$(shuf -i 1024-65535 -n 1)}
 
-echo "[$1] Forwarded QEMU guest port: $SSH_RAND_PORT->22; $NGINX_RAND_PORT->8080 $REDIS_RAND_PORT->6379" 1>&2
+echo "[$1] Forwarded QEMU guest port: $SSH_RAND_PORT->22; $NGINX_RAND_PORT->8080 $REDIS_RAND_PORT->6379 $IPERF_RAND_PORT->5201" 1>&2
 
 COMMON_QEMU_ARGS="\
     -cpu Icelake-Server,+x2apic \
@@ -23,7 +24,7 @@ COMMON_QEMU_ARGS="\
     -serial chardev:mux \
     -monitor chardev:mux \
     -chardev stdio,id=mux,mux=on,signal=off,logfile=qemu.log \
-    -netdev user,id=net01,hostfwd=tcp::$SSH_RAND_PORT-:22,hostfwd=tcp::$NGINX_RAND_PORT-:8080,hostfwd=tcp::$REDIS_RAND_PORT-:6379 \
+    -netdev user,id=net01,hostfwd=tcp::$SSH_RAND_PORT-:22,hostfwd=tcp::$NGINX_RAND_PORT-:8080,hostfwd=tcp::$REDIS_RAND_PORT-:6379,hostfwd=tcp::$IPERF_RAND_PORT-:5201 \
     -object filter-dump,id=filter0,netdev=net01,file=virtio-net.pcap \
     -device isa-debug-exit,iobase=0xf4,iosize=0x04 \
     -drive if=none,format=raw,id=x0,file=./test/build/ext2.img \