Enhance UI and functionality for benchmark

2025-06-13 07:16:47 +00:00 · 2024-08-02 13:44:43 +08:00 · 2024-08-02 13:44:43 +08:00 · c43c07c9fb
commit c43c07c9fb
parent 20a856b07f
32 changed files with 135 additions and 82 deletions
--- a/.github/workflows/benchmark_asterinas.yml
+++ b/.github/workflows/benchmark_asterinas.yml
@ -66,17 +66,28 @@ jobs:
          make install_osdk
          bash test/benchmark/bench_linux_and_aster.sh ${{ matrix.benchmark }}

-    - name: Prepare threshold values
+    - name: Set up benchmark configuration
      run: |
-        echo "Configuring thresholds..."
        ALERT_THRESHOLD=$(jq -r '.alert_threshold' test/benchmark/${{ matrix.benchmark }}/config.json)
        echo "ALERT_THRESHOLD=$ALERT_THRESHOLD" >> $GITHUB_ENV
+        
+        ALERT_TOOL=$(jq -r '.alert_tool' test/benchmark/${{ matrix.benchmark }}/config.json)
+        if [ "${ALERT_TOOL}" = "null" ]; then
+          ALERT_TOOL="customSmallerIsBetter"
+        fi
+        echo "ALERT_TOOL=$ALERT_TOOL" >> $GITHUB_ENV
+        
+        DESCRIPTION=$(jq -r '.description' test/benchmark/${{ matrix.benchmark }}/config.json)
+        if [ -z "$DESCRIPTION" ]; then
+          DESCRIPTION=""
+        fi
+        echo "DESCRIPTION=$DESCRIPTION" >> $GITHUB_ENV

    - name: Store benchmark results
-      uses: asterinas/github-action-benchmark@v1
+      uses: asterinas/github-action-benchmark@v2
      with:
        name: ${{ matrix.benchmark }} Benchmark
-        tool: 'customSmallerIsBetter'
+        tool: ${{ env.ALERT_TOOL }}
        output-file-path: result_${{ matrix.benchmark }}.json
        benchmark-data-dir-path: '' 
        github-token: ${{ secrets.BENCHMARK_SECRET }}
@ -85,3 +96,4 @@ jobs:
        alert-threshold: ${{ env.ALERT_THRESHOLD }} 
        comment-on-alert: true
        fail-on-alert: true
+        chart-description: ${{ env.DESCRIPTION }}
--- a/test/benchmark/README.md
+++ b/test/benchmark/README.md
@ -77,17 +77,23 @@ To add a new benchmark to the Asternias Continuous Integration (CI) system, foll
 2. **Create the Necessary Files:**
   - **config.json:**
     ```json
-     {
-         "alert_threshold": "125%",
-         "pattern": "Syscall average latency:",
-         "field": "4"
-     }
+      {
+        "alert_threshold": "125%",
+        "alert_tool": "customBiggerIsBetter",
+        "search_pattern": "134.22",
+        "result_index": "2",
+        "description": "The memory bandwidth for copying 128 MB of data on a single processor using the fcp (fast copy) method."
+      }
     ```
-     - `alert_threshold`: Set the threshold for alerting. If the benchmark result exceeds this threshold, an alert will be triggered.
-     - `pattern`: Define the pattern to extract the benchmark result from the output.
-     - `field`: Specify the index of the result in the extracted output.
-    
-      For example, if the benchmark output is "Syscall average latency: 1000 ns", the `pattern` is "Syscall average latency:", and the `field` is "4". `jq` will extract "1000" as the benchmark result.
+     
+    - `alert_threshold`: Set the threshold for alerting. If the benchmark result exceeds this threshold, an alert will be triggered. Note that the threshold should usually be greater than 100%.
+    - `alert_tool`: Choose the validation tool to use. The available options are `customBiggerIsBetter` and `customSmallerIsBetter`. Refer to [this](https://github.com/benchmark-action/github-action-benchmark?tab=readme-ov-file#tool-required) for more details. If using `customBiggerIsBetter`, the alert will be triggered when `prev.value / current.value` exceeds the threshold. If using `customSmallerIsBetter`, the alert will be triggered when `current.value / prev.value` exceeds the threshold.
+    - `search_pattern`: Define a regular expression to extract benchmark results from the output using `awk`. This regular expression is designed to match specific patterns in the output, effectively isolating the benchmark results and producing a set of fragments.
+    - `result_index`: Specify the index of the result in the extracted output. This field is aligned with `awk`'s action.
+    - `description`: Provide a brief description of the benchmark.
+
+    For example, if the benchmark output is "Syscall average latency: 1000 ns", the `search_pattern` is "Syscall average latency:", and the `result_index` is "4". `awk` will extract "1000" as the benchmark result. See the `awk` [manual](https://www.gnu.org/software/gawk/manual/gawk.html#Getting-Started) for more information.
+

   - **result_template.json:**
     ```json
--- a/test/benchmark/bench_linux_and_aster.sh
+++ b/test/benchmark/bench_linux_and_aster.sh
@ -33,10 +33,11 @@ EOF
    echo "$init_script"
 }

+# Run the benchmark on Linux and Asterinas
 run_benchmark() {
    local benchmark="$1"
-    local avg_pattern="$2"
-    local avg_field="$3"
+    local search_pattern="$2"
+    local result_index="$3"

    local linux_output="${BENCHMARK_DIR}/linux_output.txt"
    local aster_output="${BENCHMARK_DIR}/aster_output.txt"
@ -73,17 +74,17 @@ run_benchmark() {
    eval "$qemu_cmd"

    echo "Parsing results..."
-    local LINUX_AVG ASTER_AVG
-    LINUX_AVG=$(awk "/${avg_pattern}/{print \$$avg_field}" "${linux_output}" | tr -d '\r')
-    ASTER_AVG=$(awk "/${avg_pattern}/{print \$$avg_field}" "${aster_output}" | tr -d '\r')
+    local linux_avg aster_avg
+    linux_avg=$(awk "/${search_pattern}/{print \$$result_index}" "${linux_output}" | tr -d '\r')
+    aster_avg=$(awk "/${search_pattern}/{print \$$result_index}" "${aster_output}" | tr -d '\r')

-    if [ -z "${LINUX_AVG}" ] || [ -z "${ASTER_AVG}" ]; then
+    if [ -z "${linux_avg}" ] || [ -z "${aster_avg}" ]; then
        echo "Error: Failed to parse the average value from the benchmark output" >&2
        exit 1
    fi

    echo "Updating the result template with average values..."
-    jq --arg linux_avg "${LINUX_AVG}" --arg aster_avg "${ASTER_AVG}" \
+    jq --arg linux_avg "${linux_avg}" --arg aster_avg "${aster_avg}" \
        '(.[] | select(.extra == "linux_avg") | .value) |= $linux_avg |
         (.[] | select(.extra == "aster_avg") | .value) |= $aster_avg' \
        "${result_template}" > "${result_file}"
@ -105,9 +106,9 @@ if [ ! -d "$BENCHMARK_DIR/$BENCHMARK" ]; then
    exit 1
 fi

-PATTERN=$(jq -r '.pattern' "$BENCHMARK_DIR/$BENCHMARK/config.json")
-FIELD=$(jq -r '.field' "$BENCHMARK_DIR/$BENCHMARK/config.json")
+search_pattern=$(jq -r '.search_pattern' "$BENCHMARK_DIR/$BENCHMARK/config.json")
+result_index=$(jq -r '.result_index' "$BENCHMARK_DIR/$BENCHMARK/config.json")

-run_benchmark "$BENCHMARK" "$PATTERN" "$FIELD"
+run_benchmark "$BENCHMARK" "$search_pattern" "$result_index"

 echo "Benchmark completed successfully."
--- a/test/benchmark/lmbench-ctx/config.json
+++ b/test/benchmark/lmbench-ctx/config.json
@ -1,5 +1,7 @@
 {
    "alert_threshold": "125%",
-    "pattern": "18 ",
-    "field": "2"
+    "alert_tool": "customSmallerIsBetter",
+    "search_pattern": "18 ",
+    "result_index": "2",
+    "description": "The latency of context switching between 18 contexts on a single processor."
 }
--- a/test/benchmark/lmbench-ctx/result_template.json
+++ b/test/benchmark/lmbench-ctx/result_template.json
@ -1,13 +1,13 @@
 [
    {
        "name": "Average context switch latency on Linux",
-        "unit": "ms",
+        "unit": "µs",
        "value": 0,
        "extra": "linux_avg"
    },
    {
        "name": "Average context switch latency on Asterinas",
-        "unit": "ms",
+        "unit": "µs",
        "value": 0,
        "extra": "aster_avg"
    }
--- a/test/benchmark/lmbench-exec/config.json
+++ b/test/benchmark/lmbench-exec/config.json
@ -1,5 +1,7 @@
 {
    "alert_threshold": "125%",
-    "pattern": "Process fork\\+execve",
-    "field": "3"
+    "alert_tool": "customSmallerIsBetter",
+    "search_pattern": "Process fork\\+execve",
+    "result_index": "3",
+    "description": "The latency of creating and executing processes on a single processor."
 }
--- a/test/benchmark/lmbench-exec/result_template.json
+++ b/test/benchmark/lmbench-exec/result_template.json
@ -1,13 +1,13 @@
 [
    {
        "name": "Average exec latency on Linux",
-        "unit": "ms",
+        "unit": "µs",
        "value": 0,
        "extra": "linux_avg"
    },
    {
        "name": "Average exec latency on Asterinas",
-        "unit": "ms",
+        "unit": "µs",
        "value": 0,
        "extra": "aster_avg"
    }
--- a/test/benchmark/lmbench-fork/config.json
+++ b/test/benchmark/lmbench-fork/config.json
@ -1,5 +1,7 @@
 {
    "alert_threshold": "125%",
-    "pattern": "Process fork",
-    "field": "3"
+    "alert_tool": "customSmallerIsBetter",
+    "search_pattern": "Process fork",
+    "result_index": "3",
+    "description": "The latency of the fork system call on a single processor."
 }
--- a/test/benchmark/lmbench-fork/result_template.json
+++ b/test/benchmark/lmbench-fork/result_template.json
@ -1,13 +1,13 @@
 [
    {
        "name": "Average Fork latency on Linux",
-        "unit": "ms",
+        "unit": "µs",
        "value": 0,
        "extra": "linux_avg"
    },
    {
        "name": "Average Fork latency on Asterinas",
-        "unit": "ms",
+        "unit": "µs",
        "value": 0,
        "extra": "aster_avg"
    }
--- a/test/benchmark/lmbench-fstat/config.json
+++ b/test/benchmark/lmbench-fstat/config.json
@ -1,5 +1,7 @@
 {
    "alert_threshold": "125%",
-    "pattern": "Simple fstat",
-    "field": "3"
+    "alert_tool": "customSmallerIsBetter",
+    "search_pattern": "Simple fstat",
+    "result_index": "3",
+    "description": "The latency of the fstat system call on a single processor."
 }
--- a/test/benchmark/lmbench-fstat/result_template.json
+++ b/test/benchmark/lmbench-fstat/result_template.json
@ -1,13 +1,13 @@
 [
    {
        "name": "Average fstat latency on Linux",
-        "unit": "ms",
+        "unit": "µs",
        "value": 0,
        "extra": "linux_avg"
    },
    {
        "name": "Average fstat latency on Asterinas",
-        "unit": "ms",
+        "unit": "µs",
        "value": 0,
        "extra": "aster_avg"
    }
--- a/test/benchmark/lmbench-getpid/config.json
+++ b/test/benchmark/lmbench-getpid/config.json
@ -1,5 +1,7 @@
 {
    "alert_threshold": "125%",
-    "pattern": "Simple syscall:",
-    "field": "3"
+    "alert_tool": "customSmallerIsBetter",
+    "search_pattern": "Simple syscall:",
+    "result_index": "3",
+    "description": "The latency of the getpid system call on a single processor."
 }
--- a/test/benchmark/lmbench-getpid/result_template.json
+++ b/test/benchmark/lmbench-getpid/result_template.json
@ -1,13 +1,13 @@
 [
    {
        "name": "Average syscall latency on Linux",
-        "unit": "ms",
+        "unit": "µs",
        "value": 0,
        "extra": "linux_avg"
    },
    {
        "name": "Average syscall latency on Asterinas",
-        "unit": "ms",
+        "unit": "µs",
        "value": 0,
        "extra": "aster_avg"
    }
--- a/test/benchmark/lmbench-mem-fcp/config.json
+++ b/test/benchmark/lmbench-mem-fcp/config.json
@ -1,5 +1,7 @@
 {
    "alert_threshold": "125%",
-    "pattern": "134.22",
-    "field": "2"
+    "alert_tool": "customBiggerIsBetter",
+    "search_pattern": "134.22",
+    "result_index": "2",
+    "description": "The memory bandwidth for copying 128 MB of data on a single processor using the fcp (fast copy) method."
 }
--- a/test/benchmark/lmbench-mem-frd/config.json
+++ b/test/benchmark/lmbench-mem-frd/config.json
@ -1,5 +1,7 @@
 {
    "alert_threshold": "125%",
-    "pattern": "268.44",
-    "field": "2"
+    "alert_tool": "customBiggerIsBetter",
+    "search_pattern": "268.44",
+    "result_index": "2",
+    "description": "The memory bandwidth for reading 256 MB of data on a single processor."
 }
--- a/test/benchmark/lmbench-mem-fwr/config.json
+++ b/test/benchmark/lmbench-mem-fwr/config.json
@ -1,5 +1,7 @@
 {
    "alert_threshold": "125%",
-    "pattern": "268.44",
-    "field": "2"
+    "alert_tool": "customBiggerIsBetter",
+    "search_pattern": "268.44",
+    "result_index": "2",
+    "description": "The memory bandwidth for writing 256 MB of data on a single processor using the fwr (fast write) method."
 }
--- a/test/benchmark/lmbench-open/config.json
+++ b/test/benchmark/lmbench-open/config.json
@ -1,5 +1,7 @@
 {
    "alert_threshold": "125%",
-    "pattern": "Simple open\\/close",
-    "field": "3"
+    "alert_tool": "customSmallerIsBetter",
+    "search_pattern": "Simple open\\/close",
+    "result_index": "3",
+    "description": "The latency of the open system call on a single processor."
 }
--- a/test/benchmark/lmbench-open/result_template.json
+++ b/test/benchmark/lmbench-open/result_template.json
@ -1,13 +1,13 @@
 [
    {
        "name": "Average open latency on Linux",
-        "unit": "ms",
+        "unit": "µs",
        "value": 0,
        "extra": "linux_avg"
    },
    {
        "name": "Average open latency on Asterinas",
-        "unit": "ms",
+        "unit": "µs",
        "value": 0,
        "extra": "aster_avg"
    }
--- a/test/benchmark/lmbench-read/config.json
+++ b/test/benchmark/lmbench-read/config.json
@ -1,5 +1,7 @@
 {
    "alert_threshold": "150%",
-    "pattern": "Simple read:",
-    "field": "3"
+    "alert_tool": "customSmallerIsBetter",
+    "search_pattern": "Simple read:",
+    "result_index": "3",
+    "description": "The latency of the read system call on a single processor."
 }
--- a/test/benchmark/lmbench-read/result_template.json
+++ b/test/benchmark/lmbench-read/result_template.json
@ -1,13 +1,13 @@
 [
    {
        "name": "Average read latency on Linux",
-        "unit": "ms",
+        "unit": "µs",
        "value": 0,
        "extra": "linux_avg"
    },
    {
        "name": "Average read latency on Asterinas",
-        "unit": "ms",
+        "unit": "µs",
        "value": 0,
        "extra": "aster_avg"
    }
--- a/test/benchmark/lmbench-shell/config.json
+++ b/test/benchmark/lmbench-shell/config.json
@ -1,5 +1,7 @@
 {
    "alert_threshold": "125%",
-    "pattern": "Process fork\\+\\/bin\\/sh",
-    "field": "4"
+    "alert_tool": "customSmallerIsBetter",
+    "search_pattern": "Process fork\\+\\/bin\\/sh",
+    "result_index": "4",
+    "description": "The latency of creating and executing a shell process on a single processor."
 }
--- a/test/benchmark/lmbench-shell/result_template.json
+++ b/test/benchmark/lmbench-shell/result_template.json
@ -1,13 +1,13 @@
 [
    {
        "name": "Average shell latency on Linux",
-        "unit": "ms",
+        "unit": "µs",
        "value": 0,
        "extra": "linux_avg"
    },
    {
        "name": "Average shell latency on Asterinas",
-        "unit": "ms",
+        "unit": "µs",
        "value": 0,
        "extra": "aster_avg"
    }
--- a/test/benchmark/lmbench-signal/config.json
+++ b/test/benchmark/lmbench-signal/config.json
@ -1,5 +1,7 @@
 {
    "alert_threshold": "125%",
-    "pattern": "Signal handler overhead:",
-    "field": "4"
+    "alert_tool": "customSmallerIsBetter",
+    "search_pattern": "Signal handler overhead:",
+    "result_index": "4",
+    "description": "The latency of signal handling on a single processor."
 }
--- a/test/benchmark/lmbench-signal/result_template.json
+++ b/test/benchmark/lmbench-signal/result_template.json
@ -1,13 +1,13 @@
 [
    {
        "name": "Average Signal handler overhead on Linux",
-        "unit": "ms",
+        "unit": "µs",
        "value": 0,
        "extra": "linux_avg"
    },
    {
        "name": "Average Signal handler overhead on Asterinas",
-        "unit": "ms",
+        "unit": "µs",
        "value": 0,
        "extra": "aster_avg"
    }
--- a/test/benchmark/lmbench-stat/config.json
+++ b/test/benchmark/lmbench-stat/config.json
@ -1,5 +1,7 @@
 {
    "alert_threshold": "125%",
-    "pattern": "Simple stat",
-    "field": "3"
+    "alert_tool": "customSmallerIsBetter",
+    "search_pattern": "Simple stat",
+    "result_index": "3",
+    "description": "The latency of the stat system call on a single processor."
 }
--- a/test/benchmark/lmbench-stat/result_template.json
+++ b/test/benchmark/lmbench-stat/result_template.json
@ -1,13 +1,13 @@
 [
    {
        "name": "Average stat latency on Linux",
-        "unit": "ms",
+        "unit": "µs",
        "value": 0,
        "extra": "linux_avg"
    },
    {
        "name": "Average stat latency on Asterinas",
-        "unit": "ms",
+        "unit": "µs",
        "value": 0,
        "extra": "aster_avg"
    }
--- a/test/benchmark/lmbench-unix-latency/config.json
+++ b/test/benchmark/lmbench-unix-latency/config.json
@ -1,5 +1,7 @@
 {
    "alert_threshold": "125%",
-    "pattern": "sock stream latency",
-    "field": "5"
+    "alert_tool": "customSmallerIsBetter",
+    "search_pattern": "sock stream latency",
+    "result_index": "5",
+    "description": "The latency of UNIX domain socket communication on a single processor."
 }
--- a/test/benchmark/lmbench-unix-latency/result_template.json
+++ b/test/benchmark/lmbench-unix-latency/result_template.json
@ -1,13 +1,13 @@
 [
    {
        "name": "Average unix latency on Linux",
-        "unit": "ms",
+        "unit": "µs",
        "value": 0,
        "extra": "linux_avg"
    },
    {
        "name": "Average unix latency on Asterinas",
-        "unit": "ms",
+        "unit": "µs",
        "value": 0,
        "extra": "aster_avg"
    }
--- a/test/benchmark/lmbench-write/config.json
+++ b/test/benchmark/lmbench-write/config.json
@ -1,5 +1,7 @@
 {
    "alert_threshold": "150%",
-    "pattern": "Simple write:",
-    "field": "3"
+    "alert_tool": "customSmallerIsBetter",
+    "search_pattern": "Simple write:",
+    "result_index": "3",
+    "description": "The latency of the write system call on a single processor."
 }
--- a/test/benchmark/lmbench-write/result_template.json
+++ b/test/benchmark/lmbench-write/result_template.json
@ -1,13 +1,13 @@
 [
    {
        "name": "Average write latency on Linux",
-        "unit": "ms",
+        "unit": "µs",
        "value": 0,
        "extra": "linux_avg"
    },
    {
        "name": "Average write latency on Asterinas",
-        "unit": "ms",
+        "unit": "µs",
        "value": 0,
        "extra": "aster_avg"
    }
--- a/test/benchmark/sysbench-cpu/config.json
+++ b/test/benchmark/sysbench-cpu/config.json
@ -1,5 +1,7 @@
 {
    "alert_threshold": "130%",
-    "pattern": "avg:",
-    "field": "NF"
+    "alert_tool": "customSmallerIsBetter",
+    "search_pattern": "avg:",
+    "result_index": "NF",
+    "description": "The average time it takes for each prime number calculation to complete during the sysbench CPU test."
 }
--- a/test/benchmark/sysbench-thread/config.json
+++ b/test/benchmark/sysbench-thread/config.json
@ -1,5 +1,7 @@
 {
    "alert_threshold": "130%",
-    "pattern": "avg:",
-    "field": "NF"
+    "alert_tool": "customSmallerIsBetter",
+    "search_pattern": "avg:",
+    "result_index": "NF",
+    "description": "The average latency it takes for each thread operation (creation, yielding, locking, etc.) to complete."
 }