From 0c8200dc7b6bb8d9f7a8c319c67b842d83e25a75 Mon Sep 17 00:00:00 2001
From: jiangjianfeng <jiangjianfeng.jjf@antgroup.com>
Date: Tue, 31 Dec 2024 06:40:27 +0000
Subject: [PATCH] Add nginx benchmark and workaround

---
 .github/workflows/benchmark_asterinas.yml     |  5 ++
 kernel/libs/aster-bigtcp/src/socket/bound.rs  |  5 +-
 test/benchmark/nginx/generate_random_html.sh  | 51 +++++++++++++++++++
 .../nginx/http_file16KB_bw/bench_result.json  | 16 ++++++
 test/benchmark/nginx/http_file16KB_bw/host.sh | 24 +++++++++
 test/benchmark/nginx/http_file16KB_bw/run.sh  | 11 ++++
 .../nginx/http_file32KB_bw/bench_result.json  | 16 ++++++
 test/benchmark/nginx/http_file32KB_bw/host.sh | 24 +++++++++
 test/benchmark/nginx/http_file32KB_bw/run.sh  | 11 ++++
 .../nginx/http_file4KB_bw/bench_result.json   | 16 ++++++
 test/benchmark/nginx/http_file4KB_bw/host.sh  | 24 +++++++++
 test/benchmark/nginx/http_file4KB_bw/run.sh   | 11 ++++
 .../nginx/http_file64KB_bw/bench_result.json  | 16 ++++++
 test/benchmark/nginx/http_file64KB_bw/host.sh | 24 +++++++++
 test/benchmark/nginx/http_file64KB_bw/run.sh  | 11 ++++
 .../nginx/http_file8KB_bw/bench_result.json   | 16 ++++++
 test/benchmark/nginx/http_file8KB_bw/host.sh  | 24 +++++++++
 test/benchmark/nginx/http_file8KB_bw/run.sh   | 11 ++++
 test/benchmark/nginx/summary.json             |  7 ++-
 19 files changed, 320 insertions(+), 3 deletions(-)
 create mode 100755 test/benchmark/nginx/generate_random_html.sh
 create mode 100644 test/benchmark/nginx/http_file16KB_bw/bench_result.json
 create mode 100644 test/benchmark/nginx/http_file16KB_bw/host.sh
 create mode 100644 test/benchmark/nginx/http_file16KB_bw/run.sh
 create mode 100644 test/benchmark/nginx/http_file32KB_bw/bench_result.json
 create mode 100644 test/benchmark/nginx/http_file32KB_bw/host.sh
 create mode 100644 test/benchmark/nginx/http_file32KB_bw/run.sh
 create mode 100644 test/benchmark/nginx/http_file4KB_bw/bench_result.json
 create mode 100644 test/benchmark/nginx/http_file4KB_bw/host.sh
 create mode 100644 test/benchmark/nginx/http_file4KB_bw/run.sh
 create mode 100644 test/benchmark/nginx/http_file64KB_bw/bench_result.json
 create mode 100644 test/benchmark/nginx/http_file64KB_bw/host.sh
 create mode 100644 test/benchmark/nginx/http_file64KB_bw/run.sh
 create mode 100644 test/benchmark/nginx/http_file8KB_bw/bench_result.json
 create mode 100644 test/benchmark/nginx/http_file8KB_bw/host.sh
 create mode 100644 test/benchmark/nginx/http_file8KB_bw/run.sh

diff --git a/.github/workflows/benchmark_asterinas.yml b/.github/workflows/benchmark_asterinas.yml
index 6985bfdc..e79d8995 100644
--- a/.github/workflows/benchmark_asterinas.yml
+++ b/.github/workflows/benchmark_asterinas.yml
@@ -85,6 +85,11 @@ jobs:
           # Nginx benchmarks
           - nginx/http_req10k_conc1_bw
           - nginx/http_req10k_conc20_bw
+          - nginx/http_file4KB_bw
+          - nginx/http_file8KB_bw
+          - nginx/http_file16KB_bw
+          - nginx/http_file32KB_bw
+          - nginx/http_file64KB_bw
           # Redis benchmarks
           - redis/ping_inline_100k_conc20_rps
           - redis/ping_mbulk_100k_conc20_rps
diff --git a/kernel/libs/aster-bigtcp/src/socket/bound.rs b/kernel/libs/aster-bigtcp/src/socket/bound.rs
index b9838ba3..6edf991b 100644
--- a/kernel/libs/aster-bigtcp/src/socket/bound.rs
+++ b/kernel/libs/aster-bigtcp/src/socket/bound.rs
@@ -122,7 +122,10 @@ impl<E: Ext> RawTcpSocketExt<E> {
     /// call this method after handling non-closing user events, because the socket can never be
     /// dead if it is not closed.
     fn update_dead(&self, this: &Arc<TcpConnectionBg<E>>) {
-        if self.state() == smoltcp::socket::tcp::State::Closed {
+        // FIXME: This is a temporary workaround to mark TimeWait socket as dead.
+        if self.state() == smoltcp::socket::tcp::State::Closed
+            || self.state() == smoltcp::socket::tcp::State::TimeWait
+        {
             this.inner.is_dead.store(true, Ordering::Relaxed);
         }
 
diff --git a/test/benchmark/nginx/generate_random_html.sh b/test/benchmark/nginx/generate_random_html.sh
new file mode 100755
index 00000000..7d179b0f
--- /dev/null
+++ b/test/benchmark/nginx/generate_random_html.sh
@@ -0,0 +1,51 @@
+#!/bin/sh
+
+# SPDX-License-Identifier: MPL-2.0
+
+# Generate a html file with random contents for the given length under `/usr/local/nginx/html`
+# Usage: ./generate_random_html.sh <length>
+
+LEN=$1
+
+# Ensure LEN is numeric and reasonable
+if ! [ "$LEN" -eq "$LEN" ] || [ "$LEN" -lt 120 ]; then 
+    echo "Error: LEN must be a numeric value greater than or equal to 120"
+    exit 1
+fi
+
+DIRNAME=/usr/local/nginx/html
+FILENAME=${DIRNAME}/${LEN}bytes.html
+
+rm -f ${FILENAME}
+
+# Base HTML content
+HEADER_CONTENT="<!DOCTYPE html>
+<html>
+<head>
+<title>Sample Page</title>
+</head>
+<body>
+<h1>Hello World!</h1><p>"
+
+# Write initial content to the file
+echo "$HEADER_CONTENT" > $FILENAME
+
+# Calculate remaining length
+HEADER_LENGTH=${#HEADER_CONTENT}  # Calculate this dynamically
+FOOTER_CONTENT="</p>
+</body>
+</html>"
+CONTENT_LENGTH=$((LEN - HEADER_LENGTH - ${#FOOTER_CONTENT}-2))
+
+# Ensure the calculated CONTENT_LENGTH is positive
+if [ "$CONTENT_LENGTH" -gt 0 ]; then
+    i=0
+    while [ "$i" -lt "$CONTENT_LENGTH" ]
+    do
+        echo -n "a" >> $FILENAME
+        i=$((i + 1))
+    done
+fi
+
+# Write the footer content
+echo "$FOOTER_CONTENT" >> $FILENAME
diff --git a/test/benchmark/nginx/http_file16KB_bw/bench_result.json b/test/benchmark/nginx/http_file16KB_bw/bench_result.json
new file mode 100644
index 00000000..c64289a5
--- /dev/null
+++ b/test/benchmark/nginx/http_file16KB_bw/bench_result.json
@@ -0,0 +1,16 @@
+{
+    "alert": {
+        "threshold": "125%",
+        "bigger_is_better": true
+    },
+    "result_extraction": {
+        "search_pattern": "Requests per second: +[0-9.]+",
+        "result_index": 4
+    },
+    "chart": {
+        "title": "Nginx HTTP request performance with 16K bytes file size",
+        "description": "ab -n 10000 -c 1 http://10.0.2.15:8080/16384bytes.html",
+        "unit": "Requests per second",
+        "legend": "Average HTTP Bandwidth over virtio-net between Host Linux and Guest {system}"
+    }
+}
\ No newline at end of file
diff --git a/test/benchmark/nginx/http_file16KB_bw/host.sh b/test/benchmark/nginx/http_file16KB_bw/host.sh
new file mode 100644
index 00000000..ff6d4428
--- /dev/null
+++ b/test/benchmark/nginx/http_file16KB_bw/host.sh
@@ -0,0 +1,24 @@
+#!/bin/sh
+
+# SPDX-License-Identifier: MPL-2.0
+
+set -e
+
+# Function to stop the guest VM
+stop_guest() {
+    echo "Stopping guest VM..."
+    pgrep qemu | xargs kill
+}
+
+# Trap EXIT signal to ensure guest VM is stopped on script exit
+trap stop_guest EXIT
+
+FILESIZE=16384
+
+# Run apache bench
+echo "Warm up......"
+ab -n 20000 -c 1 http://${GUEST_SERVER_IP_ADDRESS}:8080/${FILESIZE}bytes.html >/dev/null 2>&1
+echo "Running apache bench connected to $GUEST_SERVER_IP_ADDRESS"
+ab -n 10000 -c 1 http://${GUEST_SERVER_IP_ADDRESS}:8080/${FILESIZE}bytes.html
+
+# The trap will automatically stop the guest VM when the script exits
\ No newline at end of file
diff --git a/test/benchmark/nginx/http_file16KB_bw/run.sh b/test/benchmark/nginx/http_file16KB_bw/run.sh
new file mode 100644
index 00000000..27ce776b
--- /dev/null
+++ b/test/benchmark/nginx/http_file16KB_bw/run.sh
@@ -0,0 +1,11 @@
+#!/bin/sh
+
+# SPDX-License-Identifier: MPL-2.0
+
+set -e
+
+cp /benchmark/nginx/nginx.conf /usr/local/nginx/conf/
+/benchmark/nginx/generate_random_html.sh 16384
+
+echo "Running nginx server"
+/usr/local/nginx/sbin/nginx
\ No newline at end of file
diff --git a/test/benchmark/nginx/http_file32KB_bw/bench_result.json b/test/benchmark/nginx/http_file32KB_bw/bench_result.json
new file mode 100644
index 00000000..ad4e4b6c
--- /dev/null
+++ b/test/benchmark/nginx/http_file32KB_bw/bench_result.json
@@ -0,0 +1,16 @@
+{
+    "alert": {
+        "threshold": "125%",
+        "bigger_is_better": true
+    },
+    "result_extraction": {
+        "search_pattern": "Requests per second: +[0-9.]+",
+        "result_index": 4
+    },
+    "chart": {
+        "title": "Nginx HTTP request performance with 32K bytes file size",
+        "description": "ab -n 10000 -c 1 http://10.0.2.15:8080/32768bytes.html",
+        "unit": "Requests per second",
+        "legend": "Average HTTP Bandwidth over virtio-net between Host Linux and Guest {system}"
+    }
+}
\ No newline at end of file
diff --git a/test/benchmark/nginx/http_file32KB_bw/host.sh b/test/benchmark/nginx/http_file32KB_bw/host.sh
new file mode 100644
index 00000000..aabd10e5
--- /dev/null
+++ b/test/benchmark/nginx/http_file32KB_bw/host.sh
@@ -0,0 +1,24 @@
+#!/bin/sh
+
+# SPDX-License-Identifier: MPL-2.0
+
+set -e
+
+# Function to stop the guest VM
+stop_guest() {
+    echo "Stopping guest VM..."
+    pgrep qemu | xargs kill
+}
+
+# Trap EXIT signal to ensure guest VM is stopped on script exit
+trap stop_guest EXIT
+
+FILESIZE=32768
+
+# Run apache bench
+echo "Warm up......"
+ab -n 20000 -c 1 http://${GUEST_SERVER_IP_ADDRESS}:8080/${FILESIZE}bytes.html >/dev/null 2>&1
+echo "Running apache bench connected to $GUEST_SERVER_IP_ADDRESS"
+ab -n 10000 -c 1 http://${GUEST_SERVER_IP_ADDRESS}:8080/${FILESIZE}bytes.html
+
+# The trap will automatically stop the guest VM when the script exits
\ No newline at end of file
diff --git a/test/benchmark/nginx/http_file32KB_bw/run.sh b/test/benchmark/nginx/http_file32KB_bw/run.sh
new file mode 100644
index 00000000..ea211937
--- /dev/null
+++ b/test/benchmark/nginx/http_file32KB_bw/run.sh
@@ -0,0 +1,11 @@
+#!/bin/sh
+
+# SPDX-License-Identifier: MPL-2.0
+
+set -e
+
+cp /benchmark/nginx/nginx.conf /usr/local/nginx/conf/
+/benchmark/nginx/generate_random_html.sh 32768
+
+echo "Running nginx server"
+/usr/local/nginx/sbin/nginx
\ No newline at end of file
diff --git a/test/benchmark/nginx/http_file4KB_bw/bench_result.json b/test/benchmark/nginx/http_file4KB_bw/bench_result.json
new file mode 100644
index 00000000..ff8d37e2
--- /dev/null
+++ b/test/benchmark/nginx/http_file4KB_bw/bench_result.json
@@ -0,0 +1,16 @@
+{
+    "alert": {
+        "threshold": "125%",
+        "bigger_is_better": true
+    },
+    "result_extraction": {
+        "search_pattern": "Requests per second: +[0-9.]+",
+        "result_index": 4
+    },
+    "chart": {
+        "title": "Nginx HTTP request performance with 4K bytes file size",
+        "description": "ab -n 10000 -c 1 http://10.0.2.15:8080/4096bytes.html",
+        "unit": "Requests per second",
+        "legend": "Average HTTP Bandwidth over virtio-net between Host Linux and Guest {system}"
+    }
+}
\ No newline at end of file
diff --git a/test/benchmark/nginx/http_file4KB_bw/host.sh b/test/benchmark/nginx/http_file4KB_bw/host.sh
new file mode 100644
index 00000000..7cf609bd
--- /dev/null
+++ b/test/benchmark/nginx/http_file4KB_bw/host.sh
@@ -0,0 +1,24 @@
+#!/bin/sh
+
+# SPDX-License-Identifier: MPL-2.0
+
+set -e
+
+# Function to stop the guest VM
+stop_guest() {
+    echo "Stopping guest VM..."
+    pgrep qemu | xargs kill
+}
+
+# Trap EXIT signal to ensure guest VM is stopped on script exit
+trap stop_guest EXIT
+
+FILESIZE=4096
+
+# Run apache bench
+echo "Warm up......"
+ab -n 20000 -c 1 http://${GUEST_SERVER_IP_ADDRESS}:8080/${FILESIZE}bytes.html >/dev/null 2>&1
+echo "Running apache bench connected to $GUEST_SERVER_IP_ADDRESS"
+ab -n 10000 -c 1 http://${GUEST_SERVER_IP_ADDRESS}:8080/${FILESIZE}bytes.html
+
+# The trap will automatically stop the guest VM when the script exits
\ No newline at end of file
diff --git a/test/benchmark/nginx/http_file4KB_bw/run.sh b/test/benchmark/nginx/http_file4KB_bw/run.sh
new file mode 100644
index 00000000..bb541375
--- /dev/null
+++ b/test/benchmark/nginx/http_file4KB_bw/run.sh
@@ -0,0 +1,11 @@
+#!/bin/sh
+
+# SPDX-License-Identifier: MPL-2.0
+
+set -e
+
+cp /benchmark/nginx/nginx.conf /usr/local/nginx/conf/
+/benchmark/nginx/generate_random_html.sh 4096
+
+echo "Running nginx server"
+/usr/local/nginx/sbin/nginx
\ No newline at end of file
diff --git a/test/benchmark/nginx/http_file64KB_bw/bench_result.json b/test/benchmark/nginx/http_file64KB_bw/bench_result.json
new file mode 100644
index 00000000..657e8bf8
--- /dev/null
+++ b/test/benchmark/nginx/http_file64KB_bw/bench_result.json
@@ -0,0 +1,16 @@
+{
+    "alert": {
+        "threshold": "125%",
+        "bigger_is_better": true
+    },
+    "result_extraction": {
+        "search_pattern": "Requests per second: +[0-9.]+",
+        "result_index": 4
+    },
+    "chart": {
+        "title": "Nginx HTTP request performance with 64K bytes file size",
+        "description": "ab -n 10000 -c 1 http://10.0.2.15:8080/65536bytes.html",
+        "unit": "Requests per second",
+        "legend": "Average HTTP Bandwidth over virtio-net between Host Linux and Guest {system}"
+    }
+}
\ No newline at end of file
diff --git a/test/benchmark/nginx/http_file64KB_bw/host.sh b/test/benchmark/nginx/http_file64KB_bw/host.sh
new file mode 100644
index 00000000..ec9e945e
--- /dev/null
+++ b/test/benchmark/nginx/http_file64KB_bw/host.sh
@@ -0,0 +1,24 @@
+#!/bin/sh
+
+# SPDX-License-Identifier: MPL-2.0
+
+set -e
+
+# Function to stop the guest VM
+stop_guest() {
+    echo "Stopping guest VM..."
+    pgrep qemu | xargs kill
+}
+
+# Trap EXIT signal to ensure guest VM is stopped on script exit
+trap stop_guest EXIT
+
+FILESIZE=65536
+
+# Run apache bench
+echo "Warm up......"
+ab -n 20000 -c 1 http://${GUEST_SERVER_IP_ADDRESS}:8080/${FILESIZE}bytes.html >/dev/null 2>&1
+echo "Running apache bench connected to $GUEST_SERVER_IP_ADDRESS"
+ab -n 10000 -c 1 http://${GUEST_SERVER_IP_ADDRESS}:8080/${FILESIZE}bytes.html
+
+# The trap will automatically stop the guest VM when the script exits
\ No newline at end of file
diff --git a/test/benchmark/nginx/http_file64KB_bw/run.sh b/test/benchmark/nginx/http_file64KB_bw/run.sh
new file mode 100644
index 00000000..0b295f77
--- /dev/null
+++ b/test/benchmark/nginx/http_file64KB_bw/run.sh
@@ -0,0 +1,11 @@
+#!/bin/sh
+
+# SPDX-License-Identifier: MPL-2.0
+
+set -e
+
+cp /benchmark/nginx/nginx.conf /usr/local/nginx/conf/
+/benchmark/nginx/generate_random_html.sh 65536
+
+echo "Running nginx server"
+/usr/local/nginx/sbin/nginx
\ No newline at end of file
diff --git a/test/benchmark/nginx/http_file8KB_bw/bench_result.json b/test/benchmark/nginx/http_file8KB_bw/bench_result.json
new file mode 100644
index 00000000..30520ce5
--- /dev/null
+++ b/test/benchmark/nginx/http_file8KB_bw/bench_result.json
@@ -0,0 +1,16 @@
+{
+    "alert": {
+        "threshold": "125%",
+        "bigger_is_better": true
+    },
+    "result_extraction": {
+        "search_pattern": "Requests per second: +[0-9.]+",
+        "result_index": 4
+    },
+    "chart": {
+        "title": "Nginx HTTP request performance with 8K bytes file size",
+        "description": "ab -n 10000 -c 1 http://10.0.2.15:8080/8192bytes.html",
+        "unit": "Requests per second",
+        "legend": "Average HTTP Bandwidth over virtio-net between Host Linux and Guest {system}"
+    }
+}
\ No newline at end of file
diff --git a/test/benchmark/nginx/http_file8KB_bw/host.sh b/test/benchmark/nginx/http_file8KB_bw/host.sh
new file mode 100644
index 00000000..5417bba8
--- /dev/null
+++ b/test/benchmark/nginx/http_file8KB_bw/host.sh
@@ -0,0 +1,24 @@
+#!/bin/sh
+
+# SPDX-License-Identifier: MPL-2.0
+
+set -e
+
+# Function to stop the guest VM
+stop_guest() {
+    echo "Stopping guest VM..."
+    pgrep qemu | xargs kill
+}
+
+# Trap EXIT signal to ensure guest VM is stopped on script exit
+trap stop_guest EXIT
+
+FILESIZE=8192
+
+# Run apache bench
+echo "Warm up......"
+ab -n 20000 -c 1 http://${GUEST_SERVER_IP_ADDRESS}:8080/${FILESIZE}bytes.html >/dev/null 2>&1
+echo "Running apache bench connected to $GUEST_SERVER_IP_ADDRESS"
+ab -n 10000 -c 1 http://${GUEST_SERVER_IP_ADDRESS}:8080/${FILESIZE}bytes.html
+
+# The trap will automatically stop the guest VM when the script exits
\ No newline at end of file
diff --git a/test/benchmark/nginx/http_file8KB_bw/run.sh b/test/benchmark/nginx/http_file8KB_bw/run.sh
new file mode 100644
index 00000000..6bb1a4ac
--- /dev/null
+++ b/test/benchmark/nginx/http_file8KB_bw/run.sh
@@ -0,0 +1,11 @@
+#!/bin/sh
+
+# SPDX-License-Identifier: MPL-2.0
+
+set -e
+
+cp /benchmark/nginx/nginx.conf /usr/local/nginx/conf/
+/benchmark/nginx/generate_random_html.sh 8192
+
+echo "Running nginx server"
+/usr/local/nginx/sbin/nginx
\ No newline at end of file
diff --git a/test/benchmark/nginx/summary.json b/test/benchmark/nginx/summary.json
index e4017e0d..1e0b0252 100644
--- a/test/benchmark/nginx/summary.json
+++ b/test/benchmark/nginx/summary.json
@@ -1,6 +1,9 @@
 {
     "benchmarks": [
-        "http_req10k_conc1_bw",
-        "http_req10k_conc20_bw"
+        "http_file4KB_bw",
+        "http_file8KB_bw",
+        "http_file16KB_bw",
+        "http_file32KB_bw",
+        "http_file64KB_bw"
     ]
 }
\ No newline at end of file