From cbf7a8545189ce1032049eee4a5e06c0918dccab Mon Sep 17 00:00:00 2001 From: Cautreoxit Date: Fri, 27 Dec 2024 11:08:42 +0800 Subject: [PATCH] Add Memcached into benchmark Co-authered-by: Zhang Junyang --- .github/workflows/benchmark_asterinas.yml | 4 ++++ test/Makefile | 3 +++ test/benchmark/README.md | 5 +++-- .../common/host_guest_bench_runner.sh | 3 +++ test/benchmark/memcached/summary.json | 7 +++++++ .../t16_conc64_window10k/bench_result.json | 16 ++++++++++++++ .../memcached/t16_conc64_window10k/host.sh | 20 ++++++++++++++++++ .../memcached/t16_conc64_window10k/run.sh | 8 +++++++ .../t8_conc32_window10k/bench_result.json | 16 ++++++++++++++ .../memcached/t8_conc32_window10k/host.sh | 21 +++++++++++++++++++ .../memcached/t8_conc32_window10k/run.sh | 8 +++++++ .../t8_conc32_window20k/bench_result.json | 16 ++++++++++++++ .../memcached/t8_conc32_window20k/host.sh | 21 +++++++++++++++++++ .../memcached/t8_conc32_window20k/run.sh | 8 +++++++ tools/qemu_args.sh | 5 +++-- 15 files changed, 157 insertions(+), 4 deletions(-) create mode 100644 test/benchmark/memcached/summary.json create mode 100644 test/benchmark/memcached/t16_conc64_window10k/bench_result.json create mode 100644 test/benchmark/memcached/t16_conc64_window10k/host.sh create mode 100644 test/benchmark/memcached/t16_conc64_window10k/run.sh create mode 100644 test/benchmark/memcached/t8_conc32_window10k/bench_result.json create mode 100644 test/benchmark/memcached/t8_conc32_window10k/host.sh create mode 100644 test/benchmark/memcached/t8_conc32_window10k/run.sh create mode 100644 test/benchmark/memcached/t8_conc32_window20k/bench_result.json create mode 100644 test/benchmark/memcached/t8_conc32_window20k/host.sh create mode 100644 test/benchmark/memcached/t8_conc32_window20k/run.sh diff --git a/.github/workflows/benchmark_asterinas.yml b/.github/workflows/benchmark_asterinas.yml index fceeffe1..5f291ac5 100644 --- a/.github/workflows/benchmark_asterinas.yml +++ b/.github/workflows/benchmark_asterinas.yml @@ -93,6 +93,10 @@ jobs: # SQLite benchmarks - sqlite/ext2_benchmarks - sqlite/ramfs_benchmarks + # Memcached benchmarks + - memcached/t8_conc32_window10k + - memcached/t8_conc32_window20k + - memcached/t16_conc64_window10k fail-fast: false timeout-minutes: 60 container: diff --git a/test/Makefile b/test/Makefile index f876de02..493c23f8 100644 --- a/test/Makefile +++ b/test/Makefile @@ -61,6 +61,8 @@ $(INITRAMFS)/lib/x86_64-linux-gnu: | $(VDSO_LIB) @cp -L /lib/x86_64-linux-gnu/libresolv.so.2 $@ @# required for LevelDB-db_bench_sqlite3 @cp -L /lib/x86_64-linux-gnu/libsqlite3.so $@ + @# required for memcached + @cp -L /lib/x86_64-linux-gnu/libevent-2.1.so.7 $@ @# required for VDSO @cp -L $(VDSO_LIB) $@ @@ -101,6 +103,7 @@ $(INITRAMFS)/usr/local: @mkdir -p $@ @cp -r /usr/local/nginx $@ @cp -r /usr/local/redis $@ + @cp -r /usr/local/memcached $@ .PHONY: $(INITRAMFS)/test $(INITRAMFS)/test: diff --git a/test/benchmark/README.md b/test/benchmark/README.md index d8b64515..bf8eb96f 100644 --- a/test/benchmark/README.md +++ b/test/benchmark/README.md @@ -1,6 +1,6 @@ # Asterinas Benchmark Collection -The Asterinas Benchmark Collection evaluates the performance of Asterinas in comparison to Linux across a range of benchmarking tools (e.g., LMbench, Sysbench, iPerf) and real-world applications (e.g., Nginx, Redis, SQLite). These benchmarks are conducted under various configurations, such as within a single virtual machine (VM) or between a VM and its host. +The Asterinas Benchmark Collection evaluates the performance of Asterinas in comparison to Linux across a range of benchmarking tools (e.g., LMbench, Sysbench, iPerf) and real-world applications (e.g., Nginx, Redis, SQLite, Memcached). These benchmarks are conducted under various configurations, such as within a single virtual machine (VM) or between a VM and its host. The benchmarks are run automatically on a nightly basis through continuous integration (CI) pipelines. Results, presented in clear and visually appealing figures and tables, are available [here](https://asterinas.github.io/benchmark/). @@ -8,7 +8,7 @@ The benchmarks are run automatically on a nightly basis through continuous integ ### Benchmark Suites -The benchmark collection is organized into benchmark suites, each dedicated to a specific benchmarking tool or application. These suites focus on comparing the performance of different operating systems using a particular methodology. Currently, there are seven benchmark suites, each located in its own directory: +The benchmark collection is organized into benchmark suites, each dedicated to a specific benchmarking tool or application. These suites focus on comparing the performance of different operating systems using a particular methodology. Currently, there are eight benchmark suites, each located in its own directory: - [lmbench](https://github.com/asterinas/asterinas/tree/main/test/benchmark/lmbench) - [sysbench](https://github.com/asterinas/asterinas/tree/main/test/benchmark/sysbench) @@ -17,6 +17,7 @@ The benchmark collection is organized into benchmark suites, each dedicated to a - [sqlite](https://github.com/asterinas/asterinas/tree/main/test/benchmark/sqlite) - [redis](https://github.com/asterinas/asterinas/tree/main/test/benchmark/redis) - [nginx](https://github.com/asterinas/asterinas/tree/main/test/benchmark/nginx) +- [memcached](https://github.com/asterinas/asterinas/tree/main/test/benchmark/memcached) Each suite has a corresponding web page (e.g., [LMbench results](https://asterinas.github.io/benchmark/lmbench/)) that publishes the latest performance data. At the top of each page, a summary table showcases the most recent results, configured using the `summary.json` file in the suite's directory. diff --git a/test/benchmark/common/host_guest_bench_runner.sh b/test/benchmark/common/host_guest_bench_runner.sh index 5ab1ad5e..ff88527f 100755 --- a/test/benchmark/common/host_guest_bench_runner.sh +++ b/test/benchmark/common/host_guest_bench_runner.sh @@ -32,6 +32,9 @@ elif [[ "$BENCHMARK_PATH" =~ "tcp_virtio_lat" ]]; then elif [[ "$BENCHMARK_PATH" =~ "tcp_virtio_bw" ]]; then # Persist lmbench/bw_tcp port export LMBENCH_TCP_BW_PORT=31236 +elif [[ "$BENCHMARK_PATH" =~ "memcached" ]]; then + # Persist Memcached port + export MEMCACHED_PORT=11211 fi # Function to run the benchmark diff --git a/test/benchmark/memcached/summary.json b/test/benchmark/memcached/summary.json new file mode 100644 index 00000000..8c3bb7e6 --- /dev/null +++ b/test/benchmark/memcached/summary.json @@ -0,0 +1,7 @@ +{ + "benchmarks": [ + "t8_conc32_window10k", + "t8_conc32_window20k", + "t16_conc64_window10k" + ] +} \ No newline at end of file diff --git a/test/benchmark/memcached/t16_conc64_window10k/bench_result.json b/test/benchmark/memcached/t16_conc64_window10k/bench_result.json new file mode 100644 index 00000000..21b2a0b1 --- /dev/null +++ b/test/benchmark/memcached/t16_conc64_window10k/bench_result.json @@ -0,0 +1,16 @@ +{ + "alert": { + "threshold": "125%", + "bigger_is_better": true + }, + "result_extraction": { + "search_pattern": "Run time: [0-9]+", + "result_index": 5 + }, + "chart": { + "title": "Memaslap 10% SET and 90% GET throughput with 16 client threads, 64 concurrency and 10k window", + "description": "memaslap -t 30s -T 16 -c 64 -w 10k -S 1s", + "unit": "operations per second", + "legend": "Average OPS of SET and GET over virtio-net between Host Linux and Guest {system}" + } +} \ No newline at end of file diff --git a/test/benchmark/memcached/t16_conc64_window10k/host.sh b/test/benchmark/memcached/t16_conc64_window10k/host.sh new file mode 100644 index 00000000..681eb92d --- /dev/null +++ b/test/benchmark/memcached/t16_conc64_window10k/host.sh @@ -0,0 +1,20 @@ +#!/bin/sh + +# SPDX-License-Identifier: MPL-2.0 + +set -e + +# Function to stop the guest VM +stop_guest() { + echo "Stopping guest VM..." + pgrep qemu | xargs kill +} + +# Trap EXIT signal to ensure guest VM is stopped on script exit +trap stop_guest EXIT + +# Run memaslap bench +echo "Running memaslap bench connected to $GUEST_SERVER_IP_ADDRESS" +/usr/local/benchmark/libmemcached/bin/memaslap -s $GUEST_SERVER_IP_ADDRESS:11211 -t 30s -T 16 -c 64 -w 10k -S 1s + +# The trap will automatically stop the guest VM when the script exits \ No newline at end of file diff --git a/test/benchmark/memcached/t16_conc64_window10k/run.sh b/test/benchmark/memcached/t16_conc64_window10k/run.sh new file mode 100644 index 00000000..a0ea164c --- /dev/null +++ b/test/benchmark/memcached/t16_conc64_window10k/run.sh @@ -0,0 +1,8 @@ +#!/bin/sh + +# SPDX-License-Identifier: MPL-2.0 + +set -e + +echo "Running Memcached server" +/usr/local/memcached/bin/memcached --user=root --listen=10.0.2.15 diff --git a/test/benchmark/memcached/t8_conc32_window10k/bench_result.json b/test/benchmark/memcached/t8_conc32_window10k/bench_result.json new file mode 100644 index 00000000..a66599a7 --- /dev/null +++ b/test/benchmark/memcached/t8_conc32_window10k/bench_result.json @@ -0,0 +1,16 @@ +{ + "alert": { + "threshold": "125%", + "bigger_is_better": true + }, + "result_extraction": { + "search_pattern": "Run time: [0-9]+", + "result_index": 5 + }, + "chart": { + "title": "Memaslap 10% SET and 90% GET throughput with 8 client threads, 32 concurrency and 10k window", + "description": "memaslap -t 30s -T 8 -c 32 -w 10k -S 1s", + "unit": "operations per second", + "legend": "Average OPS of SET and GET over virtio-net between Host Linux and Guest {system}" + } +} \ No newline at end of file diff --git a/test/benchmark/memcached/t8_conc32_window10k/host.sh b/test/benchmark/memcached/t8_conc32_window10k/host.sh new file mode 100644 index 00000000..dab79035 --- /dev/null +++ b/test/benchmark/memcached/t8_conc32_window10k/host.sh @@ -0,0 +1,21 @@ +#!/bin/sh + +# SPDX-License-Identifier: MPL-2.0 + +set -e + +# Function to stop the guest VM +stop_guest() { + echo "Stopping guest VM..." + pgrep qemu | xargs kill +} + +# Trap EXIT signal to ensure guest VM is stopped on script exit +trap stop_guest EXIT + +# Run memaslap bench +echo "Running memaslap bench connected to $GUEST_SERVER_IP_ADDRESS" +# -S 1s: Dump statistic information every second +/usr/local/benchmark/libmemcached/bin/memaslap -s $GUEST_SERVER_IP_ADDRESS:11211 -t 30s -T 8 -c 32 -w 10k -S 1s + +# The trap will automatically stop the guest VM when the script exits \ No newline at end of file diff --git a/test/benchmark/memcached/t8_conc32_window10k/run.sh b/test/benchmark/memcached/t8_conc32_window10k/run.sh new file mode 100644 index 00000000..a0ea164c --- /dev/null +++ b/test/benchmark/memcached/t8_conc32_window10k/run.sh @@ -0,0 +1,8 @@ +#!/bin/sh + +# SPDX-License-Identifier: MPL-2.0 + +set -e + +echo "Running Memcached server" +/usr/local/memcached/bin/memcached --user=root --listen=10.0.2.15 diff --git a/test/benchmark/memcached/t8_conc32_window20k/bench_result.json b/test/benchmark/memcached/t8_conc32_window20k/bench_result.json new file mode 100644 index 00000000..3161c3a3 --- /dev/null +++ b/test/benchmark/memcached/t8_conc32_window20k/bench_result.json @@ -0,0 +1,16 @@ +{ + "alert": { + "threshold": "125%", + "bigger_is_better": true + }, + "result_extraction": { + "search_pattern": "Run time: [0-9]+", + "result_index": 5 + }, + "chart": { + "title": "Memaslap 10% SET and 90% GET throughput with 8 client threads, 32 concurrency and 20k window", + "description": "memaslap -t 30s -T 8 -c 32 -w 20k -S 1s", + "unit": "operations per second", + "legend": "Average OPS of SET and GET over virtio-net between Host Linux and Guest {system}" + } +} \ No newline at end of file diff --git a/test/benchmark/memcached/t8_conc32_window20k/host.sh b/test/benchmark/memcached/t8_conc32_window20k/host.sh new file mode 100644 index 00000000..9aa37551 --- /dev/null +++ b/test/benchmark/memcached/t8_conc32_window20k/host.sh @@ -0,0 +1,21 @@ +#!/bin/sh + +# SPDX-License-Identifier: MPL-2.0 + +set -e + +# Function to stop the guest VM +stop_guest() { + echo "Stopping guest VM..." + pgrep qemu | xargs kill +} + +# Trap EXIT signal to ensure guest VM is stopped on script exit +trap stop_guest EXIT + +# Run memaslap bench +echo "Running memaslap bench connected to $GUEST_SERVER_IP_ADDRESS" +# -S 1s: Dump statistic information every second +/usr/local/benchmark/libmemcached/bin/memaslap -s $GUEST_SERVER_IP_ADDRESS:11211 -t 30s -T 8 -c 32 -w 20k -S 1s + +# The trap will automatically stop the guest VM when the script exits \ No newline at end of file diff --git a/test/benchmark/memcached/t8_conc32_window20k/run.sh b/test/benchmark/memcached/t8_conc32_window20k/run.sh new file mode 100644 index 00000000..a0ea164c --- /dev/null +++ b/test/benchmark/memcached/t8_conc32_window20k/run.sh @@ -0,0 +1,8 @@ +#!/bin/sh + +# SPDX-License-Identifier: MPL-2.0 + +set -e + +echo "Running Memcached server" +/usr/local/memcached/bin/memcached --user=root --listen=10.0.2.15 diff --git a/tools/qemu_args.sh b/tools/qemu_args.sh index 46a2c973..799b843a 100755 --- a/tools/qemu_args.sh +++ b/tools/qemu_args.sh @@ -24,13 +24,14 @@ REDIS_RAND_PORT=${REDIS_PORT:-$(shuf -i 1024-65535 -n 1)} IPERF_RAND_PORT=${IPERF_PORT:-$(shuf -i 1024-65535 -n 1)} LMBENCH_TCP_LAT_RAND_PORT=${LMBENCH_TCP_LAT_PORT:-$(shuf -i 1024-65535 -n 1)} LMBENCH_TCP_BW_RAND_PORT=${LMBENCH_TCP_BW_PORT:-$(shuf -i 1024-65535 -n 1)} +MEMCACHED_RAND_PORT=${MEMCACHED_PORT:-$(shuf -i 1024-65535 -n 1)} # Optional QEMU arguments. Opt in them manually if needed. # QEMU_OPT_ARG_DUMP_PACKETS="-object filter-dump,id=filter0,netdev=net01,file=virtio-net.pcap" if [ "$NETDEV" = "user" ]; then - echo "[$1] Forwarded QEMU guest port: $SSH_RAND_PORT->22; $NGINX_RAND_PORT->8080 $REDIS_RAND_PORT->6379 $IPERF_RAND_PORT->5201 $LMBENCH_TCP_LAT_RAND_PORT->31234 $LMBENCH_TCP_BW_RAND_PORT->31236" 1>&2 - NETDEV_ARGS="-netdev user,id=net01,hostfwd=tcp::$SSH_RAND_PORT-:22,hostfwd=tcp::$NGINX_RAND_PORT-:8080,hostfwd=tcp::$REDIS_RAND_PORT-:6379,hostfwd=tcp::$IPERF_RAND_PORT-:5201,hostfwd=tcp::$LMBENCH_TCP_LAT_RAND_PORT-:31234,hostfwd=tcp::$LMBENCH_TCP_BW_RAND_PORT-:31236" + echo "[$1] Forwarded QEMU guest port: $SSH_RAND_PORT->22; $NGINX_RAND_PORT->8080 $REDIS_RAND_PORT->6379 $IPERF_RAND_PORT->5201 $LMBENCH_TCP_LAT_RAND_PORT->31234 $LMBENCH_TCP_BW_RAND_PORT->31236 $MEMCACHED_RAND_PORT->11211" 1>&2 + NETDEV_ARGS="-netdev user,id=net01,hostfwd=tcp::$SSH_RAND_PORT-:22,hostfwd=tcp::$NGINX_RAND_PORT-:8080,hostfwd=tcp::$REDIS_RAND_PORT-:6379,hostfwd=tcp::$IPERF_RAND_PORT-:5201,hostfwd=tcp::$LMBENCH_TCP_LAT_RAND_PORT-:31234,hostfwd=tcp::$LMBENCH_TCP_BW_RAND_PORT-:31236,hostfwd=tcp::$MEMCACHED_RAND_PORT-:11211" VIRTIO_NET_FEATURES=",mrg_rxbuf=off,ctrl_rx=off,ctrl_rx_extra=off,ctrl_vlan=off,ctrl_vq=off,ctrl_guest_offloads=off,ctrl_mac_addr=off,event_idx=off,queue_reset=off,guest_announce=off,indirect_desc=off" elif [ "$NETDEV" = "tap" ]; then THIS_SCRIPT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )