Use summary.json to controll the overview charts of benchmarks

.github/workflows/benchmark_asterinas.yml

@@ -97,7 +97,6 @@ jobs:
         ALERT_TOOL=$(jq -r '.alert_tool' test/benchmark/${{ matrix.benchmark }}/config.json)
         TITLE=$(jq -r '.title' test/benchmark/${{ matrix.benchmark }}/config.json)
         DESCRIPTION=$(jq -r '.description' test/benchmark/${{ matrix.benchmark }}/config.json)
-        SHOW_IN_OVERVIEW=$(jq -r '.show_in_overview' test/benchmark/${{ matrix.benchmark }}/config.json)
 
         if [ "$ALERT_THRESHOLD" = "null" ]; then
           ALERT_THRESHOLD="130%"
@@ -108,26 +107,26 @@ jobs:
         if [ "$TITLE" = "null" ]; then
           TITLE="${{ matrix.benchmark }}"
         fi
-        if [ "$SHOW_IN_OVERVIEW" = "null" ]; then
-          SHOW_IN_OVERVIEW="false"
-        fi
 
         echo "ALERT_THRESHOLD=$ALERT_THRESHOLD" >> $GITHUB_ENV
         echo "ALERT_TOOL=$ALERT_TOOL" >> $GITHUB_ENV
         echo "TITLE=$TITLE" >> $GITHUB_ENV
         echo "DESCRIPTION=$DESCRIPTION" >> $GITHUB_ENV
-        echo "SHOW_IN_OVERVIEW=$SHOW_IN_OVERVIEW" >> $GITHUB_ENV
 
         BENCHMARK_NAME=$(basename "${{ matrix.benchmark }}")
         echo "BENCHMARK_NAME=$BENCHMARK_NAME" >> $GITHUB_ENV
+        BENCHMARK_STORE_DIR=$(dirname "${{ matrix.benchmark }}")
+        echo "BENCHMARK_STORE_DIR=$BENCHMARK_STORE_DIR" >> $GITHUB_ENV
+        BENCHMARK_SUMMARY=test/benchmark/${BENCHMARK_STORE_DIR}/summary.json
+        echo "BENCHMARK_SUMMARY=$BENCHMARK_SUMMARY" >> $GITHUB_ENV
 
     - name: Store benchmark results
-      uses: asterinas/github-action-benchmark@v3
+      uses: asterinas/github-action-benchmark@v4
       with:
-        name: ${{ matrix.benchmark }}
+        name: ${{ env.BENCHMARK_NAME }}
         tool: ${{ env.ALERT_TOOL }}
         output-file-path: result_${{ env.BENCHMARK_NAME }}.json
-        benchmark-data-dir-path: '' 
+        benchmark-data-dir-path: ${{ env.BENCHMARK_STORE_DIR }}
         github-token: ${{ secrets.BENCHMARK_SECRET }}
         gh-repository: 'github.com/asterinas/benchmark'
         auto-push: true
@@ -136,4 +135,4 @@ jobs:
         fail-on-alert: true
         chart-title: ${{ env.TITLE }}
         chart-description: ${{ env.DESCRIPTION }}
-        show-in-overview: ${{ env.SHOW_IN_OVERVIEW }}
+        summary-json-path: ${{ env.BENCHMARK_SUMMARY }}

test/benchmark/README.md

@@ -83,8 +83,7 @@ To add a new benchmark to the Asternias Continuous Integration (CI) system, foll
         "search_pattern": "Simple syscall:",
         "result_index": "3",
         "description": "lat_syscall null",
-        "title": "[Process] The cost of getpid",
-        "show_in_overview": "false"
+        "title": "[Process] The cost of getpid"
       } 
      ```
      
@@ -94,10 +93,21 @@ To add a new benchmark to the Asternias Continuous Integration (CI) system, foll
     - `result_index`: Specify the index of the result in the extracted output. This field is aligned with `awk`'s action.
     - `description`: Provide a brief description of the benchmark.
     - `title`: Set the title of the benchmark.
-    - `show_in_overview`: Default is true. Set to `false` to avoid displaying the benchmark in the overview results.
 
     For example, if the benchmark output is "Syscall average latency: 1000 ns", the `search_pattern` is "Syscall average latency:", and the `result_index` is "4". `awk` will extract "1000" as the benchmark result. See the `awk` [manual](https://www.gnu.org/software/gawk/manual/gawk.html#Getting-Started) for more information.
 
+    - **summary.json:**
+    ```json
+    {
+        "benchmarks": [
+            "cpu_lat",
+            "thread_lat"
+        ]
+    }
+    ```
+    - List all the benchmarks that are included in the benchmark overview. This file is used to generate the overview chart of the benchmark results. 
+    - The benchmark does not appear in the overview chart if it is not listed in this file. But it will still be included in the detailed benchmark results.
+    - The sequence of the benchmarks in this file will be the same as the sequence in the overview chart.
 
    - **result_template.json:**
      ```json

test/benchmark/lmbench/semaphore_lat/config.json

@@ -4,6 +4,5 @@
     "search_pattern": "Semaphore latency:",
     "result_index": "3",
     "description": "lat_sem",
-    "title": "[Semaphores] The cost of semop",
-    "show_in_overview": "false"
+    "title": "[Semaphores] The cost of semop"
 }

test/benchmark/lmbench/summary.json

@@ -0,0 +1,41 @@
+{
+  "benchmarks": [
+    "mem_read_bw",
+    "mem_write_bw",
+    "mem_copy_bw",
+    "mem_pagefault_lat",
+    "mem_mmap_bw",
+    "mem_mmap_lat",
+    "process_getppid_lat",
+    "process_ctx_lat",
+    "process_fork_lat",
+    "process_exec_lat",
+    "process_shell_lat",
+    "signal_install_lat",
+    "signal_catch_lat",
+    "signal_prot_lat",
+    "pipe_lat",
+    "pipe_bw",
+    "fifo_lat",
+    "unix_lat",
+    "unix_bw",
+    "unix_connect_lat",
+    "vfs_open_lat",
+    "vfs_read_lat",
+    "vfs_write_lat",
+    "vfs_stat_lat",
+    "vfs_fstat_lat",
+    "vfs_fcntl_lat",
+    "vfs_read_pagecache_bw",
+    "vfs_select_lat",
+    "ramfs_create_delete_files_0k_ops",
+    "ramfs_create_delete_files_10k_ops",
+    "ext2_copy_files_bw",
+    "tcp_loopback_bw",
+    "tcp_loopback_lat",
+    "tcp_loopback_connect_lat",
+    "tcp_loopback_select_lat",
+    "tcp_loopback_http_bw",
+    "udp_loopback_lat"
+  ]
+}

test/benchmark/sysbench/summary.json

@@ -0,0 +1,6 @@
+{
+    "benchmarks": [
+        "cpu_lat",
+        "thread_lat"
+    ]
+}