Add Memcached into benchmark

Co-authered-by: Zhang Junyang <junyang@stu.pku.edu.cn>

.github/workflows/benchmark_asterinas.yml

@@ -93,6 +93,10 @@ jobs:
           # SQLite benchmarks
           - sqlite/ext2_benchmarks
           - sqlite/ramfs_benchmarks
+          # Memcached benchmarks
+          - memcached/t8_conc32_window10k
+          - memcached/t8_conc32_window20k
+          - memcached/t16_conc64_window10k
       fail-fast: false
     timeout-minutes: 60
     container: 

test/Makefile

@@ -61,6 +61,8 @@ $(INITRAMFS)/lib/x86_64-linux-gnu: | $(VDSO_LIB)
 	@cp -L /lib/x86_64-linux-gnu/libresolv.so.2 $@
 	@# required for LevelDB-db_bench_sqlite3
 	@cp -L /lib/x86_64-linux-gnu/libsqlite3.so $@
+	@# required for memcached
+	@cp -L /lib/x86_64-linux-gnu/libevent-2.1.so.7 $@
 	@# required for VDSO
 	@cp -L $(VDSO_LIB) $@
 
@@ -101,6 +103,7 @@ $(INITRAMFS)/usr/local:
 	@mkdir -p $@
 	@cp -r /usr/local/nginx $@
 	@cp -r /usr/local/redis $@
+	@cp -r /usr/local/memcached $@
 
 .PHONY: $(INITRAMFS)/test
 $(INITRAMFS)/test:

test/benchmark/README.md

@@ -1,6 +1,6 @@
 # Asterinas Benchmark Collection
 
-The Asterinas Benchmark Collection evaluates the performance of Asterinas in comparison to Linux across a range of benchmarking tools (e.g., LMbench, Sysbench, iPerf) and real-world applications (e.g., Nginx, Redis, SQLite). These benchmarks are conducted under various configurations, such as within a single virtual machine (VM) or between a VM and its host.
+The Asterinas Benchmark Collection evaluates the performance of Asterinas in comparison to Linux across a range of benchmarking tools (e.g., LMbench, Sysbench, iPerf) and real-world applications (e.g., Nginx, Redis, SQLite, Memcached). These benchmarks are conducted under various configurations, such as within a single virtual machine (VM) or between a VM and its host.
 
 The benchmarks are run automatically on a nightly basis through continuous integration (CI) pipelines. Results, presented in clear and visually appealing figures and tables, are available [here](https://asterinas.github.io/benchmark/).
 
@@ -8,7 +8,7 @@ The benchmarks are run automatically on a nightly basis through continuous integ
 
 ### Benchmark Suites
 
-The benchmark collection is organized into benchmark suites, each dedicated to a specific benchmarking tool or application. These suites focus on comparing the performance of different operating systems using a particular methodology. Currently, there are seven benchmark suites, each located in its own directory:
+The benchmark collection is organized into benchmark suites, each dedicated to a specific benchmarking tool or application. These suites focus on comparing the performance of different operating systems using a particular methodology. Currently, there are eight benchmark suites, each located in its own directory:
 
 - [lmbench](https://github.com/asterinas/asterinas/tree/main/test/benchmark/lmbench)
 - [sysbench](https://github.com/asterinas/asterinas/tree/main/test/benchmark/sysbench)
@@ -17,6 +17,7 @@ The benchmark collection is organized into benchmark suites, each dedicated to a
 - [sqlite](https://github.com/asterinas/asterinas/tree/main/test/benchmark/sqlite)
 - [redis](https://github.com/asterinas/asterinas/tree/main/test/benchmark/redis)
 - [nginx](https://github.com/asterinas/asterinas/tree/main/test/benchmark/nginx)
+- [memcached](https://github.com/asterinas/asterinas/tree/main/test/benchmark/memcached)
 
 Each suite has a corresponding web page (e.g., [LMbench results](https://asterinas.github.io/benchmark/lmbench/)) that publishes the latest performance data. At the top of each page, a summary table showcases the most recent results, configured using the `summary.json` file in the suite's directory.
 

test/benchmark/common/host_guest_bench_runner.sh

@@ -32,6 +32,9 @@ elif [[ "$BENCHMARK_PATH" =~ "tcp_virtio_lat" ]]; then
 elif [[ "$BENCHMARK_PATH" =~ "tcp_virtio_bw" ]]; then
     # Persist lmbench/bw_tcp port
     export LMBENCH_TCP_BW_PORT=31236
+elif [[ "$BENCHMARK_PATH" =~ "memcached" ]]; then
+    # Persist Memcached port
+    export MEMCACHED_PORT=11211
 fi
 
 # Function to run the benchmark

test/benchmark/memcached/summary.json

@@ -0,0 +1,7 @@
+{
+    "benchmarks": [
+        "t8_conc32_window10k",
+        "t8_conc32_window20k",
+        "t16_conc64_window10k"
+    ]
+}

test/benchmark/memcached/t16_conc64_window10k/bench_result.json

@@ -0,0 +1,16 @@
+{
+    "alert": {
+        "threshold": "125%",
+        "bigger_is_better": true
+    },
+    "result_extraction": {
+        "search_pattern": "Run time: [0-9]+",
+        "result_index": 5
+    },
+    "chart": {
+        "title": "Memaslap 10% SET and 90% GET throughput with 16 client threads, 64 concurrency and 10k window",
+        "description": "memaslap -t 30s -T 16 -c 64 -w 10k -S 1s",
+        "unit": "operations per second",
+        "legend": "Average OPS of SET and GET over virtio-net between Host Linux and Guest {system}"
+    }
+}

test/benchmark/memcached/t16_conc64_window10k/host.sh

@@ -0,0 +1,20 @@
+#!/bin/sh
+
+# SPDX-License-Identifier: MPL-2.0
+
+set -e
+
+# Function to stop the guest VM
+stop_guest() {
+    echo "Stopping guest VM..."
+    pgrep qemu | xargs kill
+}
+
+# Trap EXIT signal to ensure guest VM is stopped on script exit
+trap stop_guest EXIT
+
+# Run memaslap bench
+echo "Running memaslap bench connected to $GUEST_SERVER_IP_ADDRESS"
+/usr/local/benchmark/libmemcached/bin/memaslap -s $GUEST_SERVER_IP_ADDRESS:11211 -t 30s -T 16 -c 64 -w 10k -S 1s
+
+# The trap will automatically stop the guest VM when the script exits

test/benchmark/memcached/t16_conc64_window10k/run.sh

@@ -0,0 +1,8 @@
+#!/bin/sh
+
+# SPDX-License-Identifier: MPL-2.0
+
+set -e
+
+echo "Running Memcached server"
+/usr/local/memcached/bin/memcached --user=root --listen=10.0.2.15

test/benchmark/memcached/t8_conc32_window10k/bench_result.json

@@ -0,0 +1,16 @@
+{
+    "alert": {
+        "threshold": "125%",
+        "bigger_is_better": true
+    },
+    "result_extraction": {
+        "search_pattern": "Run time: [0-9]+",
+        "result_index": 5
+    },
+    "chart": {
+        "title": "Memaslap 10% SET and 90% GET throughput with 8 client threads, 32 concurrency and 10k window",
+        "description": "memaslap -t 30s -T 8 -c 32 -w 10k -S 1s",
+        "unit": "operations per second",
+        "legend": "Average OPS of SET and GET over virtio-net between Host Linux and Guest {system}"
+    }
+}

test/benchmark/memcached/t8_conc32_window10k/host.sh

@@ -0,0 +1,21 @@
+#!/bin/sh
+
+# SPDX-License-Identifier: MPL-2.0
+
+set -e
+
+# Function to stop the guest VM
+stop_guest() {
+    echo "Stopping guest VM..."
+    pgrep qemu | xargs kill
+}
+
+# Trap EXIT signal to ensure guest VM is stopped on script exit
+trap stop_guest EXIT
+
+# Run memaslap bench
+echo "Running memaslap bench connected to $GUEST_SERVER_IP_ADDRESS"
+# -S 1s: Dump statistic information every second
+/usr/local/benchmark/libmemcached/bin/memaslap -s $GUEST_SERVER_IP_ADDRESS:11211 -t 30s -T 8 -c 32 -w 10k -S 1s
+
+# The trap will automatically stop the guest VM when the script exits

test/benchmark/memcached/t8_conc32_window10k/run.sh

@@ -0,0 +1,8 @@
+#!/bin/sh
+
+# SPDX-License-Identifier: MPL-2.0
+
+set -e
+
+echo "Running Memcached server"
+/usr/local/memcached/bin/memcached --user=root --listen=10.0.2.15

test/benchmark/memcached/t8_conc32_window20k/bench_result.json

@@ -0,0 +1,16 @@
+{
+    "alert": {
+        "threshold": "125%",
+        "bigger_is_better": true
+    },
+    "result_extraction": {
+        "search_pattern": "Run time: [0-9]+",
+        "result_index": 5
+    },
+    "chart": {
+        "title": "Memaslap 10% SET and 90% GET throughput with 8 client threads, 32 concurrency and 20k window",
+        "description": "memaslap -t 30s -T 8 -c 32 -w 20k -S 1s",
+        "unit": "operations per second",
+        "legend": "Average OPS of SET and GET over virtio-net between Host Linux and Guest {system}"
+    }
+}

test/benchmark/memcached/t8_conc32_window20k/host.sh

@@ -0,0 +1,21 @@
+#!/bin/sh
+
+# SPDX-License-Identifier: MPL-2.0
+
+set -e
+
+# Function to stop the guest VM
+stop_guest() {
+    echo "Stopping guest VM..."
+    pgrep qemu | xargs kill
+}
+
+# Trap EXIT signal to ensure guest VM is stopped on script exit
+trap stop_guest EXIT
+
+# Run memaslap bench
+echo "Running memaslap bench connected to $GUEST_SERVER_IP_ADDRESS"
+# -S 1s: Dump statistic information every second
+/usr/local/benchmark/libmemcached/bin/memaslap -s $GUEST_SERVER_IP_ADDRESS:11211 -t 30s -T 8 -c 32 -w 20k -S 1s
+
+# The trap will automatically stop the guest VM when the script exits

test/benchmark/memcached/t8_conc32_window20k/run.sh

@@ -0,0 +1,8 @@
+#!/bin/sh
+
+# SPDX-License-Identifier: MPL-2.0
+
+set -e
+
+echo "Running Memcached server"
+/usr/local/memcached/bin/memcached --user=root --listen=10.0.2.15

tools/qemu_args.sh

@@ -24,13 +24,14 @@ REDIS_RAND_PORT=${REDIS_PORT:-$(shuf -i 1024-65535 -n 1)}
 IPERF_RAND_PORT=${IPERF_PORT:-$(shuf -i 1024-65535 -n 1)}
 LMBENCH_TCP_LAT_RAND_PORT=${LMBENCH_TCP_LAT_PORT:-$(shuf -i 1024-65535 -n 1)}
 LMBENCH_TCP_BW_RAND_PORT=${LMBENCH_TCP_BW_PORT:-$(shuf -i 1024-65535 -n 1)}
+MEMCACHED_RAND_PORT=${MEMCACHED_PORT:-$(shuf -i 1024-65535 -n 1)}
 
 # Optional QEMU arguments. Opt in them manually if needed.
 # QEMU_OPT_ARG_DUMP_PACKETS="-object filter-dump,id=filter0,netdev=net01,file=virtio-net.pcap"
 
 if [ "$NETDEV" = "user" ]; then
-    echo "[$1] Forwarded QEMU guest port: $SSH_RAND_PORT->22; $NGINX_RAND_PORT->8080 $REDIS_RAND_PORT->6379 $IPERF_RAND_PORT->5201 $LMBENCH_TCP_LAT_RAND_PORT->31234 $LMBENCH_TCP_BW_RAND_PORT->31236" 1>&2
-    NETDEV_ARGS="-netdev user,id=net01,hostfwd=tcp::$SSH_RAND_PORT-:22,hostfwd=tcp::$NGINX_RAND_PORT-:8080,hostfwd=tcp::$REDIS_RAND_PORT-:6379,hostfwd=tcp::$IPERF_RAND_PORT-:5201,hostfwd=tcp::$LMBENCH_TCP_LAT_RAND_PORT-:31234,hostfwd=tcp::$LMBENCH_TCP_BW_RAND_PORT-:31236"
+    echo "[$1] Forwarded QEMU guest port: $SSH_RAND_PORT->22; $NGINX_RAND_PORT->8080 $REDIS_RAND_PORT->6379 $IPERF_RAND_PORT->5201 $LMBENCH_TCP_LAT_RAND_PORT->31234 $LMBENCH_TCP_BW_RAND_PORT->31236 $MEMCACHED_RAND_PORT->11211" 1>&2
+    NETDEV_ARGS="-netdev user,id=net01,hostfwd=tcp::$SSH_RAND_PORT-:22,hostfwd=tcp::$NGINX_RAND_PORT-:8080,hostfwd=tcp::$REDIS_RAND_PORT-:6379,hostfwd=tcp::$IPERF_RAND_PORT-:5201,hostfwd=tcp::$LMBENCH_TCP_LAT_RAND_PORT-:31234,hostfwd=tcp::$LMBENCH_TCP_BW_RAND_PORT-:31236,hostfwd=tcp::$MEMCACHED_RAND_PORT-:11211"
     VIRTIO_NET_FEATURES=",mrg_rxbuf=off,ctrl_rx=off,ctrl_rx_extra=off,ctrl_vlan=off,ctrl_vq=off,ctrl_guest_offloads=off,ctrl_mac_addr=off,event_idx=off,queue_reset=off,guest_announce=off,indirect_desc=off"
 elif [ "$NETDEV" = "tap" ]; then
     THIS_SCRIPT_DIR=$( cd "$( dirname "${BASH_SOURCE[0]}" )" && pwd )