From c8c9b9753e5eab280c71d38f7a928e7868b81abb Mon Sep 17 00:00:00 2001 From: Jianfeng Jiang Date: Thu, 14 Nov 2024 07:37:51 +0000 Subject: [PATCH] Make all network benchmark run with tap backend and vhost on --- .../comps/virtio/src/device/network/config.rs | 19 ++++++++++++------- .../comps/virtio/src/device/network/device.rs | 10 +++++++++- test/benchmark/bench_linux_and_aster.sh | 6 +++--- .../common/host_guest_bench_runner.sh | 3 +++ test/benchmark/iperf3/tcp_virtio_bw/host.sh | 3 ++- .../lmbench/tcp_virtio_bw_64k/host.sh | 5 +++-- test/benchmark/lmbench/tcp_virtio_lat/host.sh | 4 ++-- .../nginx/http_req10k_conc1_bw/host.sh | 3 ++- .../nginx/http_req10k_conc20_bw/host.sh | 3 ++- .../redis/get_100k_conc20_rps/host.sh | 3 ++- .../redis/ping_inline_100k_conc20_rps/host.sh | 3 ++- .../redis/ping_mbulk_100k_conc20_rps/host.sh | 5 +++-- .../redis/set_100k_conc20_rps/host.sh | 5 +++-- tools/qemu_args.sh | 4 +++- 14 files changed, 51 insertions(+), 25 deletions(-) diff --git a/kernel/comps/virtio/src/device/network/config.rs b/kernel/comps/virtio/src/device/network/config.rs index 2026361d..9c0c9e2b 100644 --- a/kernel/comps/virtio/src/device/network/config.rs +++ b/kernel/comps/virtio/src/device/network/config.rs @@ -32,13 +32,18 @@ bitflags! { const VIRTIO_NET_F_GUEST_ANNOUNCE = 1 << 21; // Driver can send gratuitous packets. const VIRTIO_NET_F_MQ = 1 << 22; // Device supports multiqueue with automatic receive steering. const VIRTIO_NET_F_CTRL_MAC_ADDR = 1 << 23; // Set MAC address through control channel. - // const VIRTIO_NET_F_HOST_USO = 1 << 56; // Device can receive USO packets. - // const VIRTIO_NET_F_HASH_REPORT = 1 << 57; // Device can report per-packet hash value and a type of calculated hash. - // const VIRTIO_NET_F_GUEST_HDRLEN = 1 << 59; // Driver can provide the exact hdr_len value. Device benefits from knowing the exact header length. - // const VIRTIO_NET_F_RSS = 1 << 60; // Device supports RSS (receive-side scaling) with Toeplitz hash calculation and configurable hash parameters for receive steering. - // const VIRTIO_NET_F_RSC_EXT = 1 << 61; // DevicecanprocessduplicatedACKsandreportnumberofcoalescedseg- ments and duplicated ACKs. - // const VIRTIO_NET_F_STANDBY = 1 << 62; // Device may act as a standby for a primary device with the same MAC address. - // const VIRTIO_NET_F_SPEED_DUPLEX = 1 << 63; // Device reports speed and duplex. + const VIRTIO_NET_F_HASH_TUNNEL = 1 << 51; // Device supports inner header hash for encapsulated packets. + const VIRTIO_NET_F_VQ_NOTF_COAL = 1 << 52; // Device supports virtqueue notification coalescing. + const VIRTIO_NET_F_NOTF_COAL = 1 << 53; // Device supports notifications coalescing. + const VIRTIO_NET_F_GUEST_USO4 = 1 << 54; // Driver can receive USOv4 packets. + const VIRTIO_NET_F_GUEST_USO6 = 1 << 55; // Driver can receive USOv6 packets. + const VIRTIO_NET_F_HOST_USO = 1 << 56; // Device can receive USO packets. + const VIRTIO_NET_F_HASH_REPORT = 1 << 57; // Device can report per-packet hash value and a type of calculated hash. + const VIRTIO_NET_F_GUEST_HDRLEN = 1 << 59; // Driver can provide the exact hdr_len value. Device benefits from knowing the exact header length. + const VIRTIO_NET_F_RSS = 1 << 60; // Device supports RSS (receive-side scaling) with Toeplitz hash calculation and configurable hash parameters for receive steering. + const VIRTIO_NET_F_RSC_EXT = 1 << 61; // DevicecanprocessduplicatedACKsandreportnumberofcoalescedseg- ments and duplicated ACKs. + const VIRTIO_NET_F_STANDBY = 1 << 62; // Device may act as a standby for a primary device with the same MAC address. + const VIRTIO_NET_F_SPEED_DUPLEX = 1 << 63; // Device reports speed and duplex. } } diff --git a/kernel/comps/virtio/src/device/network/device.rs b/kernel/comps/virtio/src/device/network/device.rs index 038d0560..2b09af89 100644 --- a/kernel/comps/virtio/src/device/network/device.rs +++ b/kernel/comps/virtio/src/device/network/device.rs @@ -10,7 +10,7 @@ use aster_network::{ AnyNetworkDevice, EthernetAddr, RxBuffer, TxBuffer, VirtioNetError, RX_BUFFER_POOL, }; use aster_util::slot_vec::SlotVec; -use log::debug; +use log::{debug, warn}; use ostd::{ mm::DmaStream, sync::{LocalIrqDisabled, SpinLock}, @@ -44,6 +44,14 @@ impl NetworkDevice { let device_features = NetworkFeatures::from_bits_truncate(device_features); let supported_features = NetworkFeatures::support_features(); let network_features = device_features & supported_features; + + if network_features != device_features { + warn!( + "Virtio net contains unsupported device features: {:?}", + device_features.difference(supported_features) + ); + } + debug!("{:?}", network_features); network_features.bits() } diff --git a/test/benchmark/bench_linux_and_aster.sh b/test/benchmark/bench_linux_and_aster.sh index d1def937..456b2824 100755 --- a/test/benchmark/bench_linux_and_aster.sh +++ b/test/benchmark/bench_linux_and_aster.sh @@ -63,7 +63,7 @@ run_benchmark() { if [ -n "$aster_scheme" ] && [ "$aster_scheme" != "null" ]; then aster_scheme_cmd="SCHEME=${aster_scheme}" fi - local asterinas_cmd="make run BENCHMARK=${benchmark} ${aster_scheme_cmd} ENABLE_KVM=1 RELEASE_LTO=1 2>&1" + local asterinas_cmd="make run BENCHMARK=${benchmark} ${aster_scheme_cmd} ENABLE_KVM=1 RELEASE_LTO=1 NETDEV=tap VHOST=on 2>&1" local linux_cmd="/usr/local/qemu/bin/qemu-system-x86_64 \ --no-reboot \ -smp 1 \ @@ -76,8 +76,8 @@ run_benchmark() { -drive if=none,format=raw,id=x0,file=${BENCHMARK_DIR}/../build/ext2.img \ -device virtio-blk-pci,bus=pcie.0,addr=0x6,drive=x0,serial=vext2,disable-legacy=on,disable-modern=off,queue-size=64,num-queues=1,request-merging=off,backend_defaults=off,discard=off,write-zeroes=off,event_idx=off,indirect_desc=off,queue_reset=off \ -append 'console=ttyS0 rdinit=/benchmark/common/bench_runner.sh ${benchmark} linux mitigations=off hugepages=0 transparent_hugepage=never quiet' \ - -netdev user,id=net01,hostfwd=tcp::5201-:5201,hostfwd=tcp::6379-:6379,hostfwd=tcp::8080-:8080,hostfwd=tcp::31234-:31234,hostfwd=tcp::31236-:31236 \ - -device virtio-net-pci,netdev=net01,disable-legacy=on,disable-modern=off,mrg_rxbuf=off,ctrl_rx=off,ctrl_rx_extra=off,ctrl_vlan=off,ctrl_vq=off,ctrl_guest_offloads=off,ctrl_mac_addr=off,event_idx=off,queue_reset=off,guest_announce=off,indirect_desc=off \ + -netdev tap,id=net01,script=${BENCHMARK_DIR}/../../tools/net/qemu-ifup.sh,downscript=${BENCHMARK_DIR}/../../tools/net/qemu-ifdown.sh,vhost=on \ + -device virtio-net-pci,netdev=net01,disable-legacy=on,disable-modern=off,csum=off,guest_csum=off,ctrl_guest_offloads=off,guest_tso4=off,guest_tso6=off,guest_ecn=off,guest_ufo=off,host_tso4=off,host_tso6=off,host_ecn=off,host_ufo=off,mrg_rxbuf=off,ctrl_vq=off,ctrl_rx=off,ctrl_vlan=off,ctrl_rx_extra=off,guest_announce=off,ctrl_mac_addr=off,host_ufo=off,guest_uso4=off,guest_uso6=off,host_uso=off \ -nographic \ 2>&1" case "${benchmark_type}" in diff --git a/test/benchmark/common/host_guest_bench_runner.sh b/test/benchmark/common/host_guest_bench_runner.sh index e06fa1a2..d0dedb96 100755 --- a/test/benchmark/common/host_guest_bench_runner.sh +++ b/test/benchmark/common/host_guest_bench_runner.sh @@ -62,6 +62,9 @@ run_benchmark() { # Sleep for a short time to ensure the guest is fully ready sleep 1 + # The listening server address in guest VM + export GUEST_SERVER_IP_ADDRESS=10.0.2.15 + # Run the host command and save the output to the specified file. bash "${BENCHMARK_PATH}/host.sh" 2>&1 | tee "${output_file}" diff --git a/test/benchmark/iperf3/tcp_virtio_bw/host.sh b/test/benchmark/iperf3/tcp_virtio_bw/host.sh index cb7439d8..9c61a2fb 100755 --- a/test/benchmark/iperf3/tcp_virtio_bw/host.sh +++ b/test/benchmark/iperf3/tcp_virtio_bw/host.sh @@ -14,6 +14,7 @@ stop_guest() { trap stop_guest EXIT # Run iperf3 client -/usr/local/benchmark/iperf/bin/iperf3 -c 127.0.0.1 -f m +echo "Running iperf3 client connected to $GUEST_SERVER_IP_ADDRESS" +/usr/local/benchmark/iperf/bin/iperf3 -c $GUEST_SERVER_IP_ADDRESS -f m # The trap will automatically stop the guest VM when the script exits \ No newline at end of file diff --git a/test/benchmark/lmbench/tcp_virtio_bw_64k/host.sh b/test/benchmark/lmbench/tcp_virtio_bw_64k/host.sh index 0f31263a..9a90914d 100644 --- a/test/benchmark/lmbench/tcp_virtio_bw_64k/host.sh +++ b/test/benchmark/lmbench/tcp_virtio_bw_64k/host.sh @@ -15,7 +15,8 @@ stop_guest() { trap stop_guest EXIT # Run lmbench tcp client -echo "Running lmbench tcp client" -/usr/local/benchmark/lmbench/bw_tcp -m 65536 -P 1 127.0.0.1 +echo "Running lmbench tcp client connected to $GUEST_SERVER_IP_ADDRESS" +# The -W parameter of bw_tcp refers to the number of bytes transferred during the warm-up phase. +/usr/local/benchmark/lmbench/bw_tcp -m 65536 -P 1 -W 50000000 -N 100 $GUEST_SERVER_IP_ADDRESS # The trap will automatically stop the guest VM when the script exits diff --git a/test/benchmark/lmbench/tcp_virtio_lat/host.sh b/test/benchmark/lmbench/tcp_virtio_lat/host.sh index 4bcb9c76..ab20001f 100644 --- a/test/benchmark/lmbench/tcp_virtio_lat/host.sh +++ b/test/benchmark/lmbench/tcp_virtio_lat/host.sh @@ -15,7 +15,7 @@ stop_guest() { trap stop_guest EXIT # Run lmbench tcp client -echo "Running lmbench tcp client" -/usr/local/benchmark/lmbench/lat_tcp -P 1 127.0.0.1 +echo "Running lmbench tcp client connected to $GUEST_SERVER_IP_ADDRESS" +/usr/local/benchmark/lmbench/lat_tcp -P 1 $GUEST_SERVER_IP_ADDRESS # The trap will automatically stop the guest VM when the script exits \ No newline at end of file diff --git a/test/benchmark/nginx/http_req10k_conc1_bw/host.sh b/test/benchmark/nginx/http_req10k_conc1_bw/host.sh index b7998446..f5bd3803 100755 --- a/test/benchmark/nginx/http_req10k_conc1_bw/host.sh +++ b/test/benchmark/nginx/http_req10k_conc1_bw/host.sh @@ -14,6 +14,7 @@ stop_guest() { trap stop_guest EXIT # Run apache bench -ab -n 10000 -c 1 http://localhost:8080/index.html +echo "Running apache bench connected to $GUEST_SERVER_IP_ADDRESS" +ab -n 10000 -c 1 http://$GUEST_SERVER_IP_ADDRESS:8080/index.html # The trap will automatically stop the guest VM when the script exits \ No newline at end of file diff --git a/test/benchmark/nginx/http_req10k_conc20_bw/host.sh b/test/benchmark/nginx/http_req10k_conc20_bw/host.sh index 309b247b..a3a7de67 100755 --- a/test/benchmark/nginx/http_req10k_conc20_bw/host.sh +++ b/test/benchmark/nginx/http_req10k_conc20_bw/host.sh @@ -14,6 +14,7 @@ stop_guest() { trap stop_guest EXIT # Run apache bench -ab -n 10000 -c 20 http://localhost:8080/index.html +echo "Running apache bench connected to $GUEST_SERVER_IP_ADDRESS" +ab -n 10000 -c 20 http://$GUEST_SERVER_IP_ADDRESS:8080/index.html # The trap will automatically stop the guest VM when the script exits \ No newline at end of file diff --git a/test/benchmark/redis/get_100k_conc20_rps/host.sh b/test/benchmark/redis/get_100k_conc20_rps/host.sh index 451403d0..753aab37 100755 --- a/test/benchmark/redis/get_100k_conc20_rps/host.sh +++ b/test/benchmark/redis/get_100k_conc20_rps/host.sh @@ -14,6 +14,7 @@ stop_guest() { trap stop_guest EXIT # Run apache bench -/usr/local/redis/bin/redis-benchmark -n 100000 -c 20 -t get +echo "Running apache bench connected to $GUEST_SERVER_IP_ADDRESS" +/usr/local/redis/bin/redis-benchmark -h $GUEST_SERVER_IP_ADDRESS -n 100000 -c 20 -t get # The trap will automatically stop the guest VM when the script exits \ No newline at end of file diff --git a/test/benchmark/redis/ping_inline_100k_conc20_rps/host.sh b/test/benchmark/redis/ping_inline_100k_conc20_rps/host.sh index fd575c66..4e2d124e 100755 --- a/test/benchmark/redis/ping_inline_100k_conc20_rps/host.sh +++ b/test/benchmark/redis/ping_inline_100k_conc20_rps/host.sh @@ -14,6 +14,7 @@ stop_guest() { trap stop_guest EXIT # Run apache bench -/usr/local/redis/bin/redis-benchmark -n 100000 -c 20 -t ping_inline +echo "Running apache bench connected to $GUEST_SERVER_IP_ADDRESS" +/usr/local/redis/bin/redis-benchmark -h $GUEST_SERVER_IP_ADDRESS -n 100000 -c 20 -t ping_inline # The trap will automatically stop the guest VM when the script exits \ No newline at end of file diff --git a/test/benchmark/redis/ping_mbulk_100k_conc20_rps/host.sh b/test/benchmark/redis/ping_mbulk_100k_conc20_rps/host.sh index f881e4b8..af651378 100755 --- a/test/benchmark/redis/ping_mbulk_100k_conc20_rps/host.sh +++ b/test/benchmark/redis/ping_mbulk_100k_conc20_rps/host.sh @@ -13,7 +13,8 @@ stop_guest() { # Trap EXIT signal to ensure guest VM is stopped on script exit trap stop_guest EXIT -# Run apache bench -/usr/local/redis/bin/redis-benchmark -n 100000 -c 20 -t ping_mbulk +# Run redis bench +echo "Running redis bench connected to $GUEST_SERVER_IP_ADDRESS" +/usr/local/redis/bin/redis-benchmark -h $GUEST_SERVER_IP_ADDRESS -n 100000 -c 20 -t ping_mbulk # The trap will automatically stop the guest VM when the script exits \ No newline at end of file diff --git a/test/benchmark/redis/set_100k_conc20_rps/host.sh b/test/benchmark/redis/set_100k_conc20_rps/host.sh index 0a514de3..84f5878c 100755 --- a/test/benchmark/redis/set_100k_conc20_rps/host.sh +++ b/test/benchmark/redis/set_100k_conc20_rps/host.sh @@ -13,7 +13,8 @@ stop_guest() { # Trap EXIT signal to ensure guest VM is stopped on script exit trap stop_guest EXIT -# Run apache bench -/usr/local/redis/bin/redis-benchmark -n 100000 -c 20 -t set +# Run redis bench +echo "Running redis bench connected to $GUEST_SERVER_IP_ADDRESS" +/usr/local/redis/bin/redis-benchmark -h $GUEST_SERVER_IP_ADDRESS -n 100000 -c 20 -t set # The trap will automatically stop the guest VM when the script exits \ No newline at end of file diff --git a/tools/qemu_args.sh b/tools/qemu_args.sh index ddc6d8f5..907b3829 100755 --- a/tools/qemu_args.sh +++ b/tools/qemu_args.sh @@ -31,11 +31,13 @@ LMBENCH_TCP_BW_RAND_PORT=${LMBENCH_TCP_BW_PORT:-$(shuf -i 1024-65535 -n 1)} if [[ "$NETDEV" =~ "user" ]]; then echo "[$1] Forwarded QEMU guest port: $SSH_RAND_PORT->22; $NGINX_RAND_PORT->8080 $REDIS_RAND_PORT->6379 $IPERF_RAND_PORT->5201 $LMBENCH_TCP_LAT_RAND_PORT->31234 $LMBENCH_TCP_BW_RAND_PORT->31236" 1>&2 NETDEV_ARGS="-netdev user,id=net01,hostfwd=tcp::$SSH_RAND_PORT-:22,hostfwd=tcp::$NGINX_RAND_PORT-:8080,hostfwd=tcp::$REDIS_RAND_PORT-:6379,hostfwd=tcp::$IPERF_RAND_PORT-:5201,hostfwd=tcp::$LMBENCH_TCP_LAT_RAND_PORT-:31234,hostfwd=tcp::$LMBENCH_TCP_BW_RAND_PORT-:31236" + VIRTIO_NET_FEATURES=",mrg_rxbuf=off,ctrl_rx=off,ctrl_rx_extra=off,ctrl_vlan=off,ctrl_vq=off,ctrl_guest_offloads=off,ctrl_mac_addr=off,event_idx=off,queue_reset=off,guest_announce=off,indirect_desc=off" elif [[ "$NETDEV" =~ "tap" ]]; then THIS_SCRIPT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd ) QEMU_IFUP_SCRIPT_PATH=$THIS_SCRIPT_DIR/net/qemu-ifup.sh QEMU_IFDOWN_SCRIPT_PATH=$THIS_SCRIPT_DIR/net/qemu-ifdown.sh NETDEV_ARGS="-netdev tap,id=net01,script=$QEMU_IFUP_SCRIPT_PATH,downscript=$QEMU_IFDOWN_SCRIPT_PATH,vhost=$VHOST" + VIRTIO_NET_FEATURES=",csum=off,guest_csum=off,ctrl_guest_offloads=off,guest_tso4=off,guest_tso6=off,guest_ecn=off,guest_ufo=off,host_tso4=off,host_tso6=off,host_ecn=off,host_ufo=off,mrg_rxbuf=off,ctrl_vq=off,ctrl_rx=off,ctrl_vlan=off,ctrl_rx_extra=off,guest_announce=off,ctrl_mac_addr=off,host_ufo=off,guest_uso4=off,guest_uso6=off,host_uso=off" else echo "Invalid netdev" 1>&2 NETDEV_ARGS="-nic none" @@ -77,7 +79,7 @@ QEMU_ARGS="\ -device virtio-blk-pci,bus=pcie.0,addr=0x6,drive=x0,serial=vext2,disable-legacy=on,disable-modern=off,queue-size=64,num-queues=1,request-merging=off,backend_defaults=off,discard=off,write-zeroes=off,event_idx=off,indirect_desc=off,queue_reset=off$IOMMU_DEV_EXTRA \ -device virtio-blk-pci,bus=pcie.0,addr=0x7,drive=x1,serial=vexfat,disable-legacy=on,disable-modern=off,queue-size=64,num-queues=1,request-merging=off,backend_defaults=off,discard=off,write-zeroes=off,event_idx=off,indirect_desc=off,queue_reset=off$IOMMU_DEV_EXTRA \ -device virtio-keyboard-pci,disable-legacy=on,disable-modern=off$IOMMU_DEV_EXTRA \ - -device virtio-net-pci,netdev=net01,disable-legacy=on,disable-modern=off,mrg_rxbuf=off,ctrl_rx=off,ctrl_rx_extra=off,ctrl_vlan=off,ctrl_vq=off,ctrl_guest_offloads=off,ctrl_mac_addr=off,event_idx=off,queue_reset=off,guest_announce=off,indirect_desc=off$IOMMU_DEV_EXTRA \ + -device virtio-net-pci,netdev=net01,disable-legacy=on,disable-modern=off$VIRTIO_NET_FEATURES$IOMMU_DEV_EXTRA \ -device virtio-serial-pci,disable-legacy=on,disable-modern=off$IOMMU_DEV_EXTRA \ -device virtconsole,chardev=mux \ $IOMMU_EXTRA_ARGS \