From a7cb71161d37cf900dc143e61f3f97fd5a35ba03 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?=E8=B4=9F=E9=9B=AA?= Date: Wed, 25 Sep 2024 06:02:11 +0000 Subject: [PATCH] Use summary.json to controll the overview charts of benchmarks --- .github/workflows/benchmark_asterinas.yml | 17 ++++---- test/benchmark/README.md | 16 ++++++-- .../lmbench/semaphore_lat/config.json | 3 +- test/benchmark/lmbench/summary.json | 41 +++++++++++++++++++ test/benchmark/sysbench/summary.json | 6 +++ 5 files changed, 69 insertions(+), 14 deletions(-) create mode 100644 test/benchmark/lmbench/summary.json create mode 100644 test/benchmark/sysbench/summary.json diff --git a/.github/workflows/benchmark_asterinas.yml b/.github/workflows/benchmark_asterinas.yml index d492558e6..85d7d7d06 100644 --- a/.github/workflows/benchmark_asterinas.yml +++ b/.github/workflows/benchmark_asterinas.yml @@ -97,7 +97,6 @@ jobs: ALERT_TOOL=$(jq -r '.alert_tool' test/benchmark/${{ matrix.benchmark }}/config.json) TITLE=$(jq -r '.title' test/benchmark/${{ matrix.benchmark }}/config.json) DESCRIPTION=$(jq -r '.description' test/benchmark/${{ matrix.benchmark }}/config.json) - SHOW_IN_OVERVIEW=$(jq -r '.show_in_overview' test/benchmark/${{ matrix.benchmark }}/config.json) if [ "$ALERT_THRESHOLD" = "null" ]; then ALERT_THRESHOLD="130%" @@ -108,26 +107,26 @@ jobs: if [ "$TITLE" = "null" ]; then TITLE="${{ matrix.benchmark }}" fi - if [ "$SHOW_IN_OVERVIEW" = "null" ]; then - SHOW_IN_OVERVIEW="false" - fi echo "ALERT_THRESHOLD=$ALERT_THRESHOLD" >> $GITHUB_ENV echo "ALERT_TOOL=$ALERT_TOOL" >> $GITHUB_ENV echo "TITLE=$TITLE" >> $GITHUB_ENV echo "DESCRIPTION=$DESCRIPTION" >> $GITHUB_ENV - echo "SHOW_IN_OVERVIEW=$SHOW_IN_OVERVIEW" >> $GITHUB_ENV BENCHMARK_NAME=$(basename "${{ matrix.benchmark }}") echo "BENCHMARK_NAME=$BENCHMARK_NAME" >> $GITHUB_ENV + BENCHMARK_STORE_DIR=$(dirname "${{ matrix.benchmark }}") + echo "BENCHMARK_STORE_DIR=$BENCHMARK_STORE_DIR" >> $GITHUB_ENV + BENCHMARK_SUMMARY=test/benchmark/${BENCHMARK_STORE_DIR}/summary.json + echo "BENCHMARK_SUMMARY=$BENCHMARK_SUMMARY" >> $GITHUB_ENV - name: Store benchmark results - uses: asterinas/github-action-benchmark@v3 + uses: asterinas/github-action-benchmark@v4 with: - name: ${{ matrix.benchmark }} + name: ${{ env.BENCHMARK_NAME }} tool: ${{ env.ALERT_TOOL }} output-file-path: result_${{ env.BENCHMARK_NAME }}.json - benchmark-data-dir-path: '' + benchmark-data-dir-path: ${{ env.BENCHMARK_STORE_DIR }} github-token: ${{ secrets.BENCHMARK_SECRET }} gh-repository: 'github.com/asterinas/benchmark' auto-push: true @@ -136,4 +135,4 @@ jobs: fail-on-alert: true chart-title: ${{ env.TITLE }} chart-description: ${{ env.DESCRIPTION }} - show-in-overview: ${{ env.SHOW_IN_OVERVIEW }} + summary-json-path: ${{ env.BENCHMARK_SUMMARY }} diff --git a/test/benchmark/README.md b/test/benchmark/README.md index 7de2da069..9f007b0a5 100644 --- a/test/benchmark/README.md +++ b/test/benchmark/README.md @@ -83,8 +83,7 @@ To add a new benchmark to the Asternias Continuous Integration (CI) system, foll "search_pattern": "Simple syscall:", "result_index": "3", "description": "lat_syscall null", - "title": "[Process] The cost of getpid", - "show_in_overview": "false" + "title": "[Process] The cost of getpid" } ``` @@ -94,10 +93,21 @@ To add a new benchmark to the Asternias Continuous Integration (CI) system, foll - `result_index`: Specify the index of the result in the extracted output. This field is aligned with `awk`'s action. - `description`: Provide a brief description of the benchmark. - `title`: Set the title of the benchmark. - - `show_in_overview`: Default is true. Set to `false` to avoid displaying the benchmark in the overview results. For example, if the benchmark output is "Syscall average latency: 1000 ns", the `search_pattern` is "Syscall average latency:", and the `result_index` is "4". `awk` will extract "1000" as the benchmark result. See the `awk` [manual](https://www.gnu.org/software/gawk/manual/gawk.html#Getting-Started) for more information. + - **summary.json:** + ```json + { + "benchmarks": [ + "cpu_lat", + "thread_lat" + ] + } + ``` + - List all the benchmarks that are included in the benchmark overview. This file is used to generate the overview chart of the benchmark results. + - The benchmark does not appear in the overview chart if it is not listed in this file. But it will still be included in the detailed benchmark results. + - The sequence of the benchmarks in this file will be the same as the sequence in the overview chart. - **result_template.json:** ```json diff --git a/test/benchmark/lmbench/semaphore_lat/config.json b/test/benchmark/lmbench/semaphore_lat/config.json index fa73a2ca1..84e7c1b33 100644 --- a/test/benchmark/lmbench/semaphore_lat/config.json +++ b/test/benchmark/lmbench/semaphore_lat/config.json @@ -4,6 +4,5 @@ "search_pattern": "Semaphore latency:", "result_index": "3", "description": "lat_sem", - "title": "[Semaphores] The cost of semop", - "show_in_overview": "false" + "title": "[Semaphores] The cost of semop" } \ No newline at end of file diff --git a/test/benchmark/lmbench/summary.json b/test/benchmark/lmbench/summary.json new file mode 100644 index 000000000..48f2106fa --- /dev/null +++ b/test/benchmark/lmbench/summary.json @@ -0,0 +1,41 @@ +{ + "benchmarks": [ + "mem_read_bw", + "mem_write_bw", + "mem_copy_bw", + "mem_pagefault_lat", + "mem_mmap_bw", + "mem_mmap_lat", + "process_getppid_lat", + "process_ctx_lat", + "process_fork_lat", + "process_exec_lat", + "process_shell_lat", + "signal_install_lat", + "signal_catch_lat", + "signal_prot_lat", + "pipe_lat", + "pipe_bw", + "fifo_lat", + "unix_lat", + "unix_bw", + "unix_connect_lat", + "vfs_open_lat", + "vfs_read_lat", + "vfs_write_lat", + "vfs_stat_lat", + "vfs_fstat_lat", + "vfs_fcntl_lat", + "vfs_read_pagecache_bw", + "vfs_select_lat", + "ramfs_create_delete_files_0k_ops", + "ramfs_create_delete_files_10k_ops", + "ext2_copy_files_bw", + "tcp_loopback_bw", + "tcp_loopback_lat", + "tcp_loopback_connect_lat", + "tcp_loopback_select_lat", + "tcp_loopback_http_bw", + "udp_loopback_lat" + ] +} \ No newline at end of file diff --git a/test/benchmark/sysbench/summary.json b/test/benchmark/sysbench/summary.json new file mode 100644 index 000000000..9c5b805e0 --- /dev/null +++ b/test/benchmark/sysbench/summary.json @@ -0,0 +1,6 @@ +{ + "benchmarks": [ + "cpu_lat", + "thread_lat" + ] +} \ No newline at end of file