Convert benchmark configuration files into YAML format

.github/workflows/benchmark_asterinas.yml

@@ -180,24 +180,29 @@ jobs:
           pattern: results_*
           path: ./results
           merge-multiple: true
+      
+      - name: Set up the environment
+        run: |
+          sudo apt-get update && sudo apt-get install -y yq
 
       - name: Set up benchmark configuration
         run: |
           BENCHMARK_DIR=$(echo ${{ matrix.benchmark }} | sed 's/-/\//g')
           BENCHMARK_SUITE=$(echo $BENCHMARK_DIR | awk -F'/' '{print $1}')
           BENCHMARK_NAME=$(echo "$BENCHMARK_DIR" | sed -E 's|^[^/]+/||; s|/bench_results||g; s|/|_|g')
-          BENCH_RESULT="test/benchmark/${BENCHMARK_DIR}/bench_result.json"
+          BENCH_RESULT="test/benchmark/${BENCHMARK_DIR}/bench_result.yaml"
-          [ -f "$BENCH_RESULT" ] || BENCH_RESULT="test/benchmark/${BENCHMARK_DIR}.json"
+          [ -f "$BENCH_RESULT" ] || BENCH_RESULT="test/benchmark/${BENCHMARK_DIR}.yaml"
-          ALERT_THRESHOLD=$(jq -r '.alert.threshold // "130%"' "$BENCH_RESULT")
+          ALERT_THRESHOLD=$(yq -r '.alert.threshold // "130%"' "$BENCH_RESULT")
-          ALERT_TOOL=$(jq -r 'if .alert.bigger_is_better == true then "customBiggerIsBetter" else "customSmallerIsBetter" end' "$BENCH_RESULT")
+          ALERT_TOOL=$(yq -r 'if (.alert.bigger_is_better == true) then "customBiggerIsBetter" else "customSmallerIsBetter" end' "$BENCH_RESULT")
-          TITLE=$(jq -r '.chart.title // "Undefined"' "$BENCH_RESULT")
+          TITLE=$(yq -r '.chart.title // "Undefined"' "$BENCH_RESULT")
-          DESCRIPTION=$(jq -r '.chart.description // "No description provided"' "$BENCH_RESULT")
+          DESCRIPTION=$(yq -r '.chart.description // "No description provided"' "$BENCH_RESULT")
           echo "BENCHMARK_SUITE=$BENCHMARK_SUITE" >> $GITHUB_ENV
           echo "BENCHMARK_NAME=$BENCHMARK_NAME" >> $GITHUB_ENV
           echo "ALERT_THRESHOLD=$ALERT_THRESHOLD" >> $GITHUB_ENV
           echo "ALERT_TOOL=$ALERT_TOOL" >> $GITHUB_ENV
           echo "TITLE=$TITLE" >> $GITHUB_ENV
           echo "DESCRIPTION=$DESCRIPTION" >> $GITHUB_ENV
+          yq . test/benchmark/$BENCHMARK_SUITE/summary.yaml > test/benchmark/$BENCHMARK_SUITE/summary.json
 
       - name: Store benchmark results
         uses: asterinas/github-action-benchmark@v4

test/benchmark/README.md

@@ -67,11 +67,11 @@ For jobs that produce a single result, the directory is structured as follows:
 ```plaintext
 <bench_suite>/
 └── <single_result_job>/
-    ├── bench_result.json
+    ├── bench_result.yaml
     └── run.sh
 ```
 
-The `bench_result.json` file contains metadata about the result, including the title, measurement unit, and whether higher or lower values indicate better performance.
+The `bench_result.yaml` file contains metadata about the result, including the title, measurement unit, and whether higher or lower values indicate better performance.
 
 #### Multi-Result Jobs
 
@@ -81,12 +81,12 @@ For jobs producing multiple results, the directory includes a `bench_results/` f
 <bench_suite>/
 └── <multi_result_job>/
     ├── bench_results/
-    │   ├── <job_a>.json
+    │   ├── <job_a>.yaml
-    │   └── <job_b>.json
+    │   └── <job_b>.yaml
     └── run.sh
 ```
 
-Each JSON file in the `bench_results/` directory describes a specific result's metadata.
+Each YAML file in the `bench_results/` directory describes a specific result's metadata.
 
 ## Adding New Benchmark Jobs
 
@@ -102,7 +102,7 @@ Each benchmark job should be added under the corresponding suite in the `test/be
 <bench_suite>/
 └── <job>/
     ├── host.sh # Only for host-guest jobs
-    ├── bench_result.json  # or bench_results/ directory for multiple results jobs
+    ├── bench_result.yaml  # or bench_results/ directory for multiple results jobs
     └── run.sh
 ```
 
@@ -134,53 +134,46 @@ iperf3 -c $GUEST_SERVER_IP_ADDRESS -f m
 
 #### Configuration Files
 
-The configuration files provide metadata about the benchmark jobs and results, such as the regression alerts, chart details, and result extraction patterns. Typically, these files are in JSON format. For single-result jobs, a `bench_result.json` file is used, while multi-result jobs have individual JSON files under `bench_results/` for each result. Some fields in these files are necessary while some are optional, depending on the benchmark's requirements. For more information, see the [`bench_result.json` format](#the-bench_resultjson-format) section.
+The configuration files provide metadata about the benchmark jobs and results, such as the regression alerts, chart details, and result extraction patterns. Typically, these files are in YAML format. For single-result jobs, a `bench_result.yaml` file is used, while multi-result jobs have individual YAML files under `bench_results/` for each result. Some fields in these files are necessary while some are optional, depending on the benchmark's requirements. For more information, see the [`bench_result.yaml` format](#the-bench_resultyaml-format) section.
 
 Below are the contents of these files for the sample benchmark:
 
-```jsonc
+```yaml
-// fio/ext2_no_iommu_seq_write_bw/bench_result.json
+# fio/ext2_no_iommu_seq_write_bw/bench_result.yaml
-{
+alert:
-    "alert": {
+  threshold: "125%"
-        "threshold": "125%",
+  bigger_is_better: true
-        "bigger_is_better": true
+
-    },
+result_extraction:
-    "result_extraction": {
+  search_pattern: "bw="
-        "search_pattern": "bw=",
+  result_index: 2
-        "result_index": 2
+
-    },
+chart:
-    "chart": {
+  title: "[Ext2] The bandwidth of sequential writes (IOMMU disabled on Asterinas)"
-        "title": "[Ext2] The bandwidth of sequential writes (IOMMU disabled on Asterinas)",
+  description: "fio -filename=/ext2/fio-test -size=1G -bs=1M -direct=1"
-        "description": "fio -filename=/ext2/fio-test -size=1G -bs=1M -direct=1",
+  unit: "MB/s"
-        "unit": "MB/s",
+  legend: "Average file write bandwidth on {system}"
-        "legend": "Average file write bandwidth on {system}"
+
-    },
+runtime_config:
-    "runtime_config": {
+  aster_scheme: "null"
-        "aster_scheme": "null"
-    }
-}  
 ```
 
-```jsonc
+```yaml
-// sqlite/ext2_benchmarks/bench_results/ext2_deletes_between.json
+# sqlite/ext2_benchmarks/bench_results/ext2_deletes_between.yaml
-{
+result_extraction:
-    "result_extraction": {
+  search_pattern: "[0-9]+ DELETEs, numeric BETWEEN, indexed...."
-        "search_pattern": "[0-9]+ DELETEs, numeric BETWEEN, indexed....",
+  result_index: 8
-        "result_index": 8
-    },
-    "chart": {
-    ...
-}
 
-// sqlite/ext2_benchmarks/bench_results/ext2_updates_between.json
+chart:
-{
+  title: "SQLite Ext2 Deletes Between"
-    "result_extraction": {
+
-        "search_pattern": "[0-9]+ UPDATES, numeric BETWEEN, indexed....",
+# sqlite/ext2_benchmarks/bench_results/ext2_updates_between.yaml
-        "result_index": 8
+result_extraction:
-    },
+  search_pattern: "[0-9]+ UPDATES, numeric BETWEEN, indexed...."
-    "chart": {
+  result_index: 8
-    ...
+
-}
+chart:
+...
 ```
 
 ### Step 3: Update Suite's `summary.json`
@@ -229,36 +222,28 @@ Secondly, we can validate modifications by running the CI pipeline on our own re
 
 Finally, if the new benchmark job runs successfully, we can commit the changes and create a pull request to merge the new benchmark into the main branch.
 
-## The `bench_result.json` Format
+## The `bench_result.yaml` Format
 
-The `bench_result.json` file configures how benchmark results are processed and displayed. Below is an example of the file to give you a big-picture understanding:
+The `bench_result.yaml` file configures how benchmark results are processed and displayed. Below is an example of the file to give you a big-picture understanding:
 
-```jsonc
+```yaml
-{
+alert:                        # Alert configuration for performance regression
-    // Configurations for performance alerts.
+  threshold: "130%"           # Acceptable performance deviation (e.g., 130% = 30% higher)
-    "alert": {
+  bigger_is_better: true      # true: Higher values are better; false: Lower values are better
-        "threshold": "130%", // Acceptable performance deviation (e.g., 130% = 30% higher).
+
-        "bigger_is_better": true // true: Higher values are better; false: Lower values are better.
+result_extraction:            # Result extraction configuration
-    },
+  search_pattern: "sender"    # Regex or string to locate results
-    // Settings for extracting benchmark results from raw outputs.
+  nth_occurrence: 1           # Optional. Which matched occurrence to use (default to 1).
-    "result_extraction": {
+  result_index: 7             # Match index to use
-        "search_pattern": "sender", // Regex or string to locate results.
+
-        "nth_occurrence": 1, // Optional. Which matched occurrence to use (default to 1).
+chart:                        # Chart configuration
-        "result_index": 7 // Match index to use.
+  title: "[Network] iperf3 sender performance using TCP"           # Title of the chart
-    },
+  description: "iperf3 -s -B 10.0.2.15"                            # Context or command associated with the benchmark
-    // Configurations for how the results are displayed in charts.
+  unit: "Mbits/sec"                                                # Measurement unit for the results
-    "chart": {
+  legend: "Average TCP Bandwidth over virtio-net between Host Linux and Guest {system}" # Chart legend with dynamic placeholder {system} supported
-        "title": "[Network] iperf3 sender performance using TCP", // Title of the chart.
+
-        "description": "iperf3 -s -B 10.0.2.15", // Context or command associated with the benchmark.
+runtime_config:              # Runtime configuration
-        "unit": "Mbits/sec", // Measurement unit for the results.
+  aster_scheme: "null"       # Corresponds to Makefile parameters, IOMMU is enabled by default (SCHEME=iommu)
-        "legend": "Average TCP Bandwidth over virtio-net between Host Linux and Guest {system}" // Chart legend with dynamic placeholder {system} supported.
-    },
-    // Optional runtime configurations for the benchmark.
-    "runtime_config": {
-        "aster_scheme": "null", // Corresponds to Makefile parameters, IOMMU is enabled by default (SCHEME=iommu).
-        "smp": 1 // Number of SMP CPUs (default to 1).
-    }
-}
 ```
 
 By adhering to this format, we ensure clarity and consistency in benchmarking workflows and reporting systems.

test/benchmark/bench_linux_and_aster.sh

@@ -6,7 +6,7 @@ set -e
 set -o pipefail
 
 # Ensure all dependencies are installed
-command -v jq >/dev/null 2>&1 || { echo >&2 "jq is not installed. Aborting."; exit 1; }
+command -v yq >/dev/null 2>&1 || command -v yq >/dev/null 2>&1 || { echo >&2 "tools are not installed. Aborting."; exit 1; }
 
 # Set up paths
 BENCHMARK_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")" &>/dev/null && pwd)"
@@ -32,10 +32,11 @@ parse_raw_results() {
     fi
 
     # Write the results into the template
-    jq --arg linux_result "${linux_result}" --arg aster_result "${aster_result}" \
+    yq --arg linux_result "${linux_result}" --arg aster_result "${aster_result}" \
         '(.[] | select(.extra == "linux_result") | .value) |= $linux_result |
          (.[] | select(.extra == "aster_result") | .value) |= $aster_result' \
         "${RESULT_TEMPLATE}" > "${result_file}"
+    echo "Results written to ${result_file}"
 }
 
 # Generate a new result template based on unit and legend
@@ -48,7 +49,7 @@ generate_template() {
     local asterinas_legend=${legend//"{system}"/"Asterinas"}
 
     # Generate the result template JSON
-    jq -n --arg linux "$linux_legend" --arg aster "$asterinas_legend" --arg unit "$unit" '[
+    yq -n --arg linux "$linux_legend" --arg aster "$asterinas_legend" --arg unit "$unit" '[
         { "name": $linux, "unit": $unit, "value": 0, "extra": "linux_result" },
         { "name": $aster, "unit": $unit, "value": 0, "extra": "aster_result" }
     ]' > "${RESULT_TEMPLATE}"
@@ -66,7 +67,8 @@ extract_result_file() {
         local second_part=$(dirname "$bench_result" | awk -F"/benchmark/$first_dir/" '{print $2}' | cut -d'/' -f1)
         echo "result_${first_dir}-${second_part}.json"
     else
-        echo "result_${relative_path//\//-}"
+        local result_file="result_${relative_path//\//-}"
+        echo "${result_file/.yaml/.json}"
     fi
 }
 
@@ -138,11 +140,11 @@ run_benchmark() {
 parse_results() {
     local bench_result="$1"
 
-    local search_pattern=$(jq -r '.result_extraction.search_pattern // empty' "$bench_result")
+    local search_pattern=$(yq -r '.result_extraction.search_pattern // empty' "$bench_result")
-    local nth_occurrence=$(jq -r '.result_extraction.nth_occurrence // 1' "$bench_result")
+    local nth_occurrence=$(yq -r '.result_extraction.nth_occurrence // 1' "$bench_result")
-    local result_index=$(jq -r '.result_extraction.result_index // empty' "$bench_result")
+    local result_index=$(yq -r '.result_extraction.result_index // empty' "$bench_result")
-    local unit=$(jq -r '.chart.unit // empty' "$bench_result")
+    local unit=$(yq -r '.chart.unit // empty' "$bench_result")
-    local legend=$(jq -r '.chart.legend // {system}' "$bench_result")
+    local legend=$(yq -r '.chart.legend // {system}' "$bench_result")
 
     generate_template "$unit" "$legend"
     parse_raw_results "$search_pattern" "$nth_occurrence" "$result_index" "$(extract_result_file "$bench_result")"
@@ -167,22 +169,22 @@ main() {
     local run_mode="guest_only"
     [[ -f "${BENCHMARK_ROOT}/${benchmark}/host.sh" ]] && run_mode="host_guest"
 
-    local bench_result="${BENCHMARK_ROOT}/${benchmark}/bench_result.json"
+    local bench_result="${BENCHMARK_ROOT}/${benchmark}/bench_result.yaml"
     local aster_scheme
     if [[ -f "$bench_result" ]]; then
-        aster_scheme=$(jq -r '.runtime_config.aster_scheme // ""' "$bench_result")
+        aster_scheme=$(yq -r '.runtime_config.aster_scheme // ""' "$bench_result")
     else
         for job in "${BENCHMARK_ROOT}/${benchmark}"/bench_results/*; do
-            [[ -f "$job" ]] && aster_scheme=$(jq -r '.runtime_config.aster_scheme // ""' "$job") && break
+            [[ -f "$job" ]] && aster_scheme=$(yq -r '.runtime_config.aster_scheme // ""' "$job") && break
         done
     fi
 
     local smp
     if [[ -f "$bench_result" ]]; then
-        smp=$(jq -r '.runtime_config.smp // 1' "$bench_result")
+        smp=$(yq -r '.runtime_config.smp // 1' "$bench_result")
     else
         for job in "${BENCHMARK_ROOT}/${benchmark}"/bench_results/*; do
-            [[ -f "$job" ]] && smp=$(jq -r '.runtime_config.smp // 1' "$job") && break
+            [[ -f "$job" ]] && smp=$(yq -r '.runtime_config.smp // 1' "$job") && break
         done
     fi
 

test/benchmark/fio/ext2_seq_read_bw/bench_result.json

@@ -1,16 +0,0 @@
-{
-    "alert": {
-        "threshold": "125%",
-        "bigger_is_better": true
-    },
-    "result_extraction": {
-        "search_pattern": "bw=[0-9]+",
-        "result_index": 2
-    },
-    "chart": {
-        "title": "[Ext2] The bandwidth of sequential reads",
-        "description": "fio -filename=/ext2/fio-test -size=1G -bs=1M -direct=1",
-        "unit": "MB/s",
-        "legend": "Average file read bandwidth on {system}"
-    }
-}

test/benchmark/fio/ext2_seq_read_bw/bench_result.yaml

@@ -0,0 +1,11 @@
+alert:
+  bigger_is_better: true
+  threshold: 125%
+chart:
+  description: fio -filename=/ext2/fio-test -size=1G -bs=1M -direct=1
+  legend: Average file read bandwidth on {system}
+  title: '[Ext2] The bandwidth of sequential reads'
+  unit: MB/s
+result_extraction:
+  result_index: 2
+  search_pattern: bw=[0-9]+

test/benchmark/fio/ext2_seq_read_bw_no_iommu/bench_result.json

@@ -1,19 +0,0 @@
-{
-    "alert": {
-        "threshold": "125%",
-        "bigger_is_better": true
-    },
-    "result_extraction": {
-        "search_pattern": "bw=[0-9]+",
-        "result_index": 2
-    },
-    "chart": {
-        "title": "[Ext2] The bandwidth of sequential reads (IOMMU disabled on Asterinas)",
-        "description": "fio -filename=/ext2/fio-test -size=1G -bs=1M -direct=1",
-        "unit": "MB/s",
-        "legend": "Average file read bandwidth on {system}"
-    },
-    "runtime_config": {
-        "aster_scheme": "null"
-    }
-}

test/benchmark/fio/ext2_seq_read_bw_no_iommu/bench_result.yaml

@@ -0,0 +1,13 @@
+alert:
+  bigger_is_better: true
+  threshold: 125%
+chart:
+  description: fio -filename=/ext2/fio-test -size=1G -bs=1M -direct=1
+  legend: Average file read bandwidth on {system}
+  title: '[Ext2] The bandwidth of sequential reads (IOMMU disabled on Asterinas)'
+  unit: MB/s
+result_extraction:
+  result_index: 2
+  search_pattern: bw=[0-9]+
+runtime_config:
+  aster_scheme: 'null'

test/benchmark/fio/ext2_seq_write_bw/bench_result.json

@@ -1,16 +0,0 @@
-{
-    "alert": {
-        "threshold": "125%",
-        "bigger_is_better": true
-    },
-    "result_extraction": {
-        "search_pattern": "bw=[0-9]+",
-        "result_index": 2
-    },
-    "chart": {
-        "title": "[Ext2] The bandwidth of sequential writes",
-        "description": "fio -filename=/ext2/fio-test -size=1G -bs=1M -direct=1",
-        "unit": "MB/s",
-        "legend": "Average file write bandwidth on {system}"
-    }
-}

test/benchmark/fio/ext2_seq_write_bw/bench_result.yaml

@@ -0,0 +1,11 @@
+alert:
+  bigger_is_better: true
+  threshold: 125%
+chart:
+  description: fio -filename=/ext2/fio-test -size=1G -bs=1M -direct=1
+  legend: Average file write bandwidth on {system}
+  title: '[Ext2] The bandwidth of sequential writes'
+  unit: MB/s
+result_extraction:
+  result_index: 2
+  search_pattern: bw=[0-9]+

test/benchmark/fio/ext2_seq_write_bw_no_iommu/bench_result.json

@@ -1,19 +0,0 @@
-{
-    "alert": {
-        "threshold": "125%",
-        "bigger_is_better": true
-    },
-    "result_extraction": {
-        "search_pattern": "bw=[0-9]+",
-        "result_index": 2
-    },
-    "chart": {
-        "title": "[Ext2] The bandwidth of sequential writes (IOMMU disabled on Asterinas)",
-        "description": "fio -filename=/ext2/fio-test -size=1G -bs=1M -direct=1",
-        "unit": "MB/s",
-        "legend": "Average file write bandwidth on {system}"
-    },
-    "runtime_config": {
-        "aster_scheme": "null"
-    }
-}

test/benchmark/fio/ext2_seq_write_bw_no_iommu/bench_result.yaml

@@ -0,0 +1,13 @@
+alert:
+  bigger_is_better: true
+  threshold: 125%
+chart:
+  description: fio -filename=/ext2/fio-test -size=1G -bs=1M -direct=1
+  legend: Average file write bandwidth on {system}
+  title: '[Ext2] The bandwidth of sequential writes (IOMMU disabled on Asterinas)'
+  unit: MB/s
+result_extraction:
+  result_index: 2
+  search_pattern: bw=[0-9]+
+runtime_config:
+  aster_scheme: 'null'

test/benchmark/fio/summary.json

@@ -1,8 +0,0 @@
-{
-    "benchmarks": [
-        "ext2_seq_write_bw",
-        "ext2_seq_read_bw",
-        "ext2_seq_write_bw_no_iommu",
-        "ext2_seq_read_bw_no_iommu"
-    ]
-}

test/benchmark/fio/summary.yaml

@@ -0,0 +1,5 @@
+benchmarks:
+  - ext2_seq_write_bw
+  - ext2_seq_read_bw
+  - ext2_seq_write_bw_no_iommu
+  - ext2_seq_read_bw_no_iommu

test/benchmark/hackbench/group8_smp1/bench_result.json

@@ -1,16 +0,0 @@
-{
-    "alert": {
-        "threshold": "130%",
-        "bigger_is_better": false
-    },
-    "result_extraction": {
-        "search_pattern": "Time:",
-        "result_index": 2
-    },
-    "chart": {
-        "title": "[Scheduler] Turnaround time of 8 groups using 1 CPU",
-        "description": "hackbench -g 8 -l 1000 -p -T",
-        "unit": "sec",
-        "legend": "Turnaround time of hackbench on {system}"
-    }
-}

test/benchmark/hackbench/group8_smp1/bench_result.yaml

@@ -0,0 +1,11 @@
+alert:
+  bigger_is_better: false
+  threshold: 130%
+chart:
+  description: hackbench -g 8 -l 1000 -p -T
+  legend: Turnaround time of hackbench on {system}
+  title: '[Scheduler] Turnaround time of 8 groups using 1 CPU'
+  unit: sec
+result_extraction:
+  result_index: 2
+  search_pattern: 'Time:'

test/benchmark/hackbench/group8_smp16/bench_result.json

@@ -1,19 +0,0 @@
-{
-    "alert": {
-        "threshold": "130%",
-        "bigger_is_better": false
-    },
-    "result_extraction": {
-        "search_pattern": "Time:",
-        "result_index": 2
-    },
-    "chart": {
-        "title": "[Scheduler] Turnaround time of 8 groups using 16 CPU",
-        "description": "hackbench -g 8 -l 1000 -p -T",
-        "unit": "sec",
-        "legend": "Turnaround time of hackbench on {system}"
-    },
-    "runtime_config": {
-        "smp": 16
-    }
-}

test/benchmark/hackbench/group8_smp16/bench_result.yaml

@@ -0,0 +1,13 @@
+alert:
+  bigger_is_better: false
+  threshold: 130%
+chart:
+  description: hackbench -g 8 -l 1000 -p -T
+  legend: Turnaround time of hackbench on {system}
+  title: '[Scheduler] Turnaround time of 8 groups using 16 CPU'
+  unit: sec
+result_extraction:
+  result_index: 2
+  search_pattern: 'Time:'
+runtime_config:
+  smp: 16

test/benchmark/hackbench/group8_smp8/bench_result.json

@@ -1,19 +0,0 @@
-{
-    "alert": {
-        "threshold": "130%",
-        "bigger_is_better": false
-    },
-    "result_extraction": {
-        "search_pattern": "Time:",
-        "result_index": 2
-    },
-    "chart": {
-        "title": "[Scheduler] Turnaround time of 8 groups using 8 CPU",
-        "description": "hackbench -g 8 -l 1000 -p -T",
-        "unit": "sec",
-        "legend": "Turnaround time of hackbench on {system}"
-    },
-    "runtime_config": {
-        "smp": 8
-    }
-}

test/benchmark/hackbench/group8_smp8/bench_result.yaml

@@ -0,0 +1,13 @@
+alert:
+  bigger_is_better: false
+  threshold: 130%
+chart:
+  description: hackbench -g 8 -l 1000 -p -T
+  legend: Turnaround time of hackbench on {system}
+  title: '[Scheduler] Turnaround time of 8 groups using 8 CPU'
+  unit: sec
+result_extraction:
+  result_index: 2
+  search_pattern: 'Time:'
+runtime_config:
+  smp: 8

test/benchmark/hackbench/summary.json

@@ -1,4 +0,0 @@
-{
-    "benchmarks": [
-    ]
-}

test/benchmark/hackbench/summary.yaml

@@ -0,0 +1 @@
+benchmarks: []

test/benchmark/iperf3/summary.json

@@ -1,5 +0,0 @@
-{
-    "benchmarks": [
-        "tcp_virtio_bw"
-    ]
-}

test/benchmark/iperf3/summary.yaml

@@ -0,0 +1,2 @@
+benchmarks:
+  - tcp_virtio_bw

test/benchmark/iperf3/tcp_virtio_bw/bench_result.json

@@ -1,16 +0,0 @@
-{
-    "alert": {
-        "threshold": "130%",
-        "bigger_is_better": true
-    },
-    "result_extraction": {
-        "search_pattern": "sender",
-        "result_index": 7
-    },
-    "chart": {
-        "title": "[Network] iperf3 sender performance using TCP (virtio-net)",
-        "description": "iperf3 -s -B 10.0.2.15",
-        "unit": "Mbits/sec",
-        "legend": "Average TCP Bandwidth over virtio-net between Host Linux and Guest {system}"
-    }
-}

test/benchmark/iperf3/tcp_virtio_bw/bench_result.yaml

@@ -0,0 +1,11 @@
+alert:
+  bigger_is_better: true
+  threshold: 130%
+chart:
+  description: iperf3 -s -B 10.0.2.15
+  legend: Average TCP Bandwidth over virtio-net between Host Linux and Guest {system}
+  title: '[Network] iperf3 sender performance using TCP (virtio-net)'
+  unit: Mbits/sec
+result_extraction:
+  result_index: 7
+  search_pattern: sender

test/benchmark/lmbench/ext2_copy_files_bw/bench_result.json

@@ -1,16 +0,0 @@
-{
-    "alert": {
-        "threshold": "125%",
-        "bigger_is_better": true
-    },
-    "result_extraction": {
-        "search_pattern": "lmdd result: +[0-9.]+",
-        "result_index": 8
-    },
-    "chart": {
-        "title": "[Ext2] The bandwidth of copying data between files",
-        "description": "lmdd",
-        "unit": "MB/s",
-        "legend": "Average file copy bandwidth on {system}"
-    }
-}

test/benchmark/lmbench/ext2_copy_files_bw/bench_result.yaml

@@ -0,0 +1,11 @@
+alert:
+  bigger_is_better: true
+  threshold: 125%
+chart:
+  description: lmdd
+  legend: Average file copy bandwidth on {system}
+  title: '[Ext2] The bandwidth of copying data between files'
+  unit: MB/s
+result_extraction:
+  result_index: 8
+  search_pattern: 'lmdd result: +[0-9.]+'

test/benchmark/lmbench/ext2_create_delete_files_0k_ops/bench_result.json

@@ -1,16 +0,0 @@
-{
-    "alert": {
-        "threshold": "125%",
-        "bigger_is_better": true
-    },
-    "result_extraction": {
-        "search_pattern": "^0k\t[0-9]+",
-        "result_index": 2
-    },
-    "chart": {
-        "title": "[Ext2] The throughput of creating/deleting small files (0KB)",
-        "description": "lat_fs -s 0k /ext2",
-        "unit": "number",
-        "legend": "Number of created/deleted files on {system}"
-    }
-}

test/benchmark/lmbench/ext2_create_delete_files_0k_ops/bench_result.yaml

@@ -0,0 +1,11 @@
+alert:
+  bigger_is_better: true
+  threshold: 125%
+chart:
+  description: lat_fs -s 0k /ext2
+  legend: Number of created/deleted files on {system}
+  title: '[Ext2] The throughput of creating/deleting small files (0KB)'
+  unit: number
+result_extraction:
+  result_index: 2
+  search_pattern: ^0k\t[0-9]+

test/benchmark/lmbench/ext2_create_delete_files_10k_ops/bench_result.json

@@ -1,16 +0,0 @@
-{
-    "alert": {
-        "threshold": "125%",
-        "bigger_is_better": true
-    },
-    "result_extraction": {
-        "search_pattern": "^10k\t[0-9]+",
-        "result_index": 2
-    },
-    "chart": {
-        "title": "[Ext2] The throughput of creating/deleting small files (10KB)",
-        "description": "lat_fs -s 10K /ext2",
-        "unit": "number",
-        "legend": "Number of created/deleted files on {system}"
-    }
-}

test/benchmark/lmbench/ext2_create_delete_files_10k_ops/bench_result.yaml

@@ -0,0 +1,11 @@
+alert:
+  bigger_is_better: true
+  threshold: 125%
+chart:
+  description: lat_fs -s 10K /ext2
+  legend: Number of created/deleted files on {system}
+  title: '[Ext2] The throughput of creating/deleting small files (10KB)'
+  unit: number
+result_extraction:
+  result_index: 2
+  search_pattern: ^0k\t[0-9]+

test/benchmark/lmbench/fifo_lat/bench_result.json

@@ -1,16 +0,0 @@
-{
-    "alert": {
-        "threshold": "125%",
-        "bigger_is_better": false
-    },
-    "result_extraction": {
-        "search_pattern": "Fifo latency: +[0-9.]+",
-        "result_index": 3
-    },
-    "chart": {
-        "title": "[FIFO] The cost of write+read (1B)",
-        "description": "lat_fifo",
-        "unit": "\u00b5s",
-        "legend": "Average fifo latency on {system}"
-    }
-}

test/benchmark/lmbench/fifo_lat/bench_result.yaml

@@ -0,0 +1,11 @@
+alert:
+  bigger_is_better: false
+  threshold: 125%
+chart:
+  description: lat_fifo
+  legend: Average fifo latency on {system}
+  title: '[FIFO] The cost of write+read (1B)'
+  unit: "\xB5s"   # Unicode for microsecond
+result_extraction:
+  result_index: 3
+  search_pattern: 'Fifo latency: +[0-9.]+'

test/benchmark/lmbench/mem_copy_bw/bench_result.json

@@ -1,16 +0,0 @@
-{
-    "alert": {
-        "threshold": "125%",
-        "bigger_is_better": true
-    },
-    "result_extraction": {
-        "search_pattern": "536.87 +[0-9.]+",
-        "result_index": 2
-    },
-    "chart": {
-        "title": "[Memory] The bandwidth of copying integers",
-        "description": "bw_mem fcp",
-        "unit": "MB/s",
-        "legend": "Average memory copy bandwidth on {system}"
-    }
-}

test/benchmark/lmbench/mem_copy_bw/bench_result.yaml

@@ -0,0 +1,11 @@
+alert:
+  bigger_is_better: true
+  threshold: 125%
+chart:
+  description: bw_mem fcp
+  legend: Average memory copy bandwidth on {system}
+  title: '[Memory] The bandwidth of copying integers'
+  unit: MB/s
+result_extraction:
+  result_index: 2
+  search_pattern: 536.87 +[0-9.]+

test/benchmark/lmbench/mem_mmap_bw/bench_result.json

@@ -1,16 +0,0 @@
-{
-    "alert": {
-        "threshold": "125%",
-        "bigger_is_better": true
-    },
-    "result_extraction": {
-        "search_pattern": "268.44 +[0-9.]+",
-        "result_index": 2
-    },
-    "chart": {
-        "title": "[Memory] The bandwidth of mmap",
-        "description": "bw_mmap",
-        "unit": "MB/s",
-        "legend": "Average mmap bandwidth on {system}"
-    }
-}

test/benchmark/lmbench/mem_mmap_bw/bench_result.yaml

@@ -0,0 +1,11 @@
+alert:
+  bigger_is_better: true
+  threshold: 125%
+chart:
+  description: bw_mmap
+  legend: Average mmap bandwidth on {system}
+  title: '[Memory] The bandwidth of mmap'
+  unit: MB/s
+result_extraction:
+  result_index: 2
+  search_pattern: 268.44 +[0-9.]+

test/benchmark/lmbench/mem_mmap_lat/bench_result.json

@@ -1,16 +0,0 @@
-{
-    "alert": {
-        "threshold": "125%",
-        "bigger_is_better": false
-    },
-    "result_extraction": {
-        "search_pattern": "4.194304 +[0-9.]+",
-        "result_index": 2
-    },
-    "chart": {
-        "title": "[Memory] The cost of mmap+unmap",
-        "description": "lat_mmap",
-        "unit": "\u00b5s",
-        "legend": "Average mmap latency on {system}"
-    }
-}

test/benchmark/lmbench/mem_mmap_lat/bench_result.yaml

@@ -0,0 +1,11 @@
+alert:
+  bigger_is_better: false
+  threshold: 125%
+chart:
+  description: lat_mmap
+  legend: Average mmap latency on {system}
+  title: '[Memory] The cost of mmap+unmap'
+  unit: "\xB5s"   # Unicode for microsecond
+result_extraction:
+  result_index: 2
+  search_pattern: 4.194304 +[0-9.]+

test/benchmark/lmbench/mem_pagefault_lat/bench_result.json

@@ -1,16 +0,0 @@
-{
-    "alert": {
-        "threshold": "125%",
-        "bigger_is_better": false
-    },
-    "result_extraction": {
-        "search_pattern": "Pagefaults on [^:]+: +[0-9.]+",
-        "result_index": 4
-    },
-    "chart": {
-        "title": "[Memory] The cost of page fault handling",
-        "description": "lat_pagefault",
-        "unit": "\u00b5s",
-        "legend": "Average page fault latency on {system}"
-    }
-}

test/benchmark/lmbench/mem_pagefault_lat/bench_result.yaml

@@ -0,0 +1,11 @@
+alert:
+  bigger_is_better: false
+  threshold: 125%
+chart:
+  description: lat_pagefault
+  legend: Average page fault latency on {system}
+  title: '[Memory] The cost of page fault handling'
+  unit: "\xB5s"   # Unicode for microsecond
+result_extraction:
+  result_index: 4
+  search_pattern: 'Pagefaults on [^:]+: +[0-9.]+'

test/benchmark/lmbench/mem_read_bw/bench_result.json

@@ -1,16 +0,0 @@
-{
-    "alert": {
-        "threshold": "125%",
-        "bigger_is_better": true
-    },
-    "result_extraction": {
-        "search_pattern": "536.87 +[0-9.]+",
-        "result_index": 2
-    },
-    "chart": {
-        "title": "[Memory] The bandwidth of reading integers",
-        "description": "bw_mem frd",
-        "unit": "MB/s",
-        "legend": "Average memory read bandwidth on {system}"
-    }
-}

test/benchmark/lmbench/mem_read_bw/bench_result.yaml

@@ -0,0 +1,11 @@
+alert:
+  bigger_is_better: true
+  threshold: 125%
+chart:
+  description: bw_mem frd
+  legend: Average memory read bandwidth on {system}
+  title: '[Memory] The bandwidth of reading integers'
+  unit: MB/s
+result_extraction:
+  result_index: 2
+  search_pattern: 536.87 +[0-9.]+

test/benchmark/lmbench/mem_write_bw/bench_result.json

@@ -1,16 +0,0 @@
-{
-    "alert": {
-        "threshold": "125%",
-        "bigger_is_better": true
-    },
-    "result_extraction": {
-        "search_pattern": "536.87 +[0-9.]+",
-        "result_index": 2
-    },
-    "chart": {
-        "title": "[Memory] The bandwidth of writing integers",
-        "description": "bw_mem fwr",
-        "unit": "MB/s",
-        "legend": "Average memory write bandwidth on {system}"
-    }
-}

test/benchmark/lmbench/mem_write_bw/bench_result.yaml

@@ -0,0 +1,11 @@
+alert:
+  bigger_is_better: true
+  threshold: 125%
+chart:
+  description: bw_mem fwr
+  legend: Average memory write bandwidth on {system}
+  title: '[Memory] The bandwidth of writing integers'
+  unit: MB/s
+result_extraction:
+  result_index: 2
+  search_pattern: 536.87 +[0-9.]+

test/benchmark/lmbench/pipe_bw/bench_result.json

@@ -1,16 +0,0 @@
-{
-    "alert": {
-        "threshold": "125%",
-        "bigger_is_better": true
-    },
-    "result_extraction": {
-        "search_pattern": "Pipe bandwidth: +[0-9.]+",
-        "result_index": 3
-    },
-    "chart": {
-        "title": "[Pipes] The bandwidth",
-        "description": "bw_pipe",
-        "unit": "MB/s",
-        "legend": "Average pipe bandwidth on {system}"
-    }
-}

test/benchmark/lmbench/pipe_bw/bench_result.yaml

@@ -0,0 +1,11 @@
+alert:
+  bigger_is_better: true
+  threshold: 125%
+chart:
+  description: bw_pipe
+  legend: Average pipe bandwidth on {system}
+  title: '[Pipes] The bandwidth'
+  unit: MB/s
+result_extraction:
+  result_index: 3
+  search_pattern: 'Pipe bandwidth: +[0-9.]+'

test/benchmark/lmbench/pipe_lat/bench_result.json

@@ -1,16 +0,0 @@
-{
-    "alert": {
-        "threshold": "125%",
-        "bigger_is_better": false
-    },
-    "result_extraction": {
-        "search_pattern": "Pipe latency: +[0-9.]+",
-        "result_index": 3
-    },
-    "chart": {
-        "title": "[Pipes] The cost of write+read (1B)",
-        "description": "lat_pipe",
-        "unit": "\u00b5s",
-        "legend": "Average pipe latency on {system}"
-    }
-}

test/benchmark/lmbench/pipe_lat/bench_result.yaml

@@ -0,0 +1,11 @@
+alert:
+  bigger_is_better: false
+  threshold: 125%
+chart:
+  description: lat_pipe
+  legend: Average pipe latency on {system}
+  title: '[Pipes] The cost of write+read (1B)'
+  unit: "\xB5s"   # Unicode for microsecond
+result_extraction:
+  result_index: 3
+  search_pattern: 'Pipe latency: +[0-9.]+'

test/benchmark/lmbench/process_ctx_lat/bench_result.json

@@ -1,16 +0,0 @@
-{
-    "alert": {
-        "threshold": "125%",
-        "bigger_is_better": false
-    },
-    "result_extraction": {
-        "search_pattern": "18 +[0-9.]+",
-        "result_index": 2
-    },
-    "chart": {
-        "title": "[Process] The cost of context switching",
-        "description": "lat_ctx 2",
-        "unit": "\u00b5s",
-        "legend": "Average context switch latency on {system}"
-    }
-}

test/benchmark/lmbench/process_ctx_lat/bench_result.yaml

@@ -0,0 +1,11 @@
+alert:
+  bigger_is_better: false
+  threshold: 125%
+chart:
+  description: lat_ctx 2
+  legend: Average context switch latency on {system}
+  title: '[Process] The cost of context switching'
+  unit: "\xB5s"   # Unicode for microsecond
+result_extraction:
+  result_index: 2
+  search_pattern: 18 +[0-9.]+

test/benchmark/lmbench/process_exec_lat/bench_result.json

@@ -1,16 +0,0 @@
-{
-    "alert": {
-        "threshold": "125%",
-        "bigger_is_better": false
-    },
-    "result_extraction": {
-        "search_pattern": "Process fork\\+execve: +[0-9.]+",
-        "result_index": 3
-    },
-    "chart": {
-        "title": "[Process] The cost of fork+exec+exit",
-        "description": "lat_proc exec",
-        "unit": "\u00b5s",
-        "legend": "Average exec latency on {system}"
-    }
-}

test/benchmark/lmbench/process_exec_lat/bench_result.yaml

@@ -0,0 +1,11 @@
+alert:
+  bigger_is_better: false
+  threshold: 125%
+chart:
+  description: lat_proc exec
+  legend: Average exec latency on {system}
+  title: '[Process] The cost of fork+exec+exit'
+  unit: "\xB5s"   # Unicode for microsecond
+result_extraction:
+  result_index: 3
+  search_pattern: 'Process fork\+execve: +[0-9.]+'

test/benchmark/lmbench/process_fork_lat/bench_result.json

@@ -1,16 +0,0 @@
-{
-    "alert": {
-        "threshold": "125%",
-        "bigger_is_better": false
-    },
-    "result_extraction": {
-        "search_pattern": "Process fork\\+exit: +[0-9.]+",
-        "result_index": 3
-    },
-    "chart": {
-        "title": "[Process] The cost of fork+exit",
-        "description": "lat_proc fork",
-        "unit": "\u00b5s",
-        "legend": "Average Fork latency on {system}"
-    }
-}

test/benchmark/lmbench/process_fork_lat/bench_result.yaml

@@ -0,0 +1,11 @@
+alert:
+  bigger_is_better: false
+  threshold: 125%
+chart:
+  description: lat_proc fork
+  legend: Average Fork latency on {system}
+  title: '[Process] The cost of fork+exit'
+  unit: "\xB5s"   # Unicode for microsecond
+result_extraction:
+  result_index: 3
+  search_pattern: 'Process fork\+exit: +[0-9.]+'

test/benchmark/lmbench/process_getppid_lat/bench_result.json

@@ -1,16 +0,0 @@
-{
-    "alert": {
-        "threshold": "125%",
-        "bigger_is_better": false
-    },
-    "result_extraction": {
-        "search_pattern": "Simple syscall: +[0-9.]+",
-        "result_index": 3
-    },
-    "chart": {
-        "title": "[Process] The cost of getppid",
-        "description": "lat_syscall null",
-        "unit": "\u00b5s",
-        "legend": "Average syscall latency on {system}"
-    }
-}

test/benchmark/lmbench/process_getppid_lat/bench_result.yaml

@@ -0,0 +1,11 @@
+alert:
+  bigger_is_better: false
+  threshold: 125%
+chart:
+  description: lat_syscall null
+  legend: Average syscall latency on {system}
+  title: '[Process] The cost of getppid'
+  unit: "\xB5s"   # Unicode for microsecond
+result_extraction:
+  result_index: 3
+  search_pattern: 'Simple syscall: +[0-9.]+'

test/benchmark/lmbench/process_shell_lat/bench_result.json

@@ -1,16 +0,0 @@
-{
-    "alert": {
-        "threshold": "125%",
-        "bigger_is_better": false
-    },
-    "result_extraction": {
-        "search_pattern": "Process fork\\+\\/bin\\/sh",
-        "result_index": 4
-    },
-    "chart": {
-        "title": "[Process] The cost of fork+exec+shell+exit",
-        "description": "lat_proc shell",
-        "unit": "\u00b5s",
-        "legend": "Average shell latency on {system}"
-    }
-}

test/benchmark/lmbench/process_shell_lat/bench_result.yaml

@@ -0,0 +1,11 @@
+alert:
+  bigger_is_better: false
+  threshold: 125%
+chart:
+  description: lat_proc shell
+  legend: Average shell latency on {system}
+  title: '[Process] The cost of fork+exec+shell+exit'
+  unit: "\xB5s"   # Unicode for microsecond
+result_extraction:
+  result_index: 4
+  search_pattern: Process fork\+\/bin\/sh

test/benchmark/lmbench/ramfs_copy_files_bw/bench_result.json

@@ -1,16 +0,0 @@
-{
-    "alert": {
-        "threshold": "125%",
-        "bigger_is_better": true
-    },
-    "result_extraction": {
-        "search_pattern": "lmdd result:",
-        "result_index": 8
-    },
-    "chart": {
-        "title": "[Ramfs] The bandwidth of copying data between files",
-        "description": "lmdd",
-        "unit": "MB/s",
-        "legend": "Average file copy bandwidth on {system}"
-    }
-}

test/benchmark/lmbench/ramfs_copy_files_bw/bench_result.yaml

@@ -0,0 +1,11 @@
+alert:
+  bigger_is_better: true
+  threshold: 125%
+chart:
+  description: lmdd
+  legend: Average file copy bandwidth on {system}
+  title: '[Ramfs] The bandwidth of copying data between files'
+  unit: MB/s
+result_extraction:
+  result_index: 8
+  search_pattern: 'lmdd result:'

test/benchmark/lmbench/ramfs_create_delete_files_0k_ops/bench_result.json

@@ -1,16 +0,0 @@
-{
-    "alert": {
-        "threshold": "125%",
-        "bigger_is_better": true
-    },
-    "result_extraction": {
-        "search_pattern": "^0k\t[0-9]+",
-        "result_index": 2
-    },
-    "chart": {
-        "title": "[Ramfs] The throughput of creating/deleting small files (0KB)",
-        "description": "lat_fs -s 0k",
-        "unit": "number",
-        "legend": "Number of created/deleted files on {system}"
-    }
-}

test/benchmark/lmbench/ramfs_create_delete_files_0k_ops/bench_result.yaml

@@ -0,0 +1,11 @@
+alert:
+  bigger_is_better: true
+  threshold: 125%
+chart:
+  description: lat_fs -s 0k
+  legend: Number of created/deleted files on {system}
+  title: '[Ramfs] The throughput of creating/deleting small files (0KB)'
+  unit: number
+result_extraction:
+  result_index: 2
+  search_pattern: ^0k

test/benchmark/lmbench/ramfs_create_delete_files_10k_ops/bench_result.json

@@ -1,16 +0,0 @@
-{
-    "alert": {
-        "threshold": "125%",
-        "bigger_is_better": true
-    },
-    "result_extraction": {
-        "search_pattern": "^10k\t[0-9]+",
-        "result_index": 2
-    },
-    "chart": {
-        "title": "[Ramfs] The throughput of creating/deleting small files (10KB)",
-        "description": "lat_fs -s 10K",
-        "unit": "number",
-        "legend": "Number of created/deleted files on {system}"
-    }
-}

test/benchmark/lmbench/ramfs_create_delete_files_10k_ops/bench_result.yaml

@@ -0,0 +1,11 @@
+alert:
+  bigger_is_better: true
+  threshold: 125%
+chart:
+  description: lat_fs -s 10K
+  legend: Number of created/deleted files on {system}
+  title: '[Ramfs] The throughput of creating/deleting small files (10KB)'
+  unit: number
+result_extraction:
+  result_index: 2
+  search_pattern: 10k

test/benchmark/lmbench/semaphore_lat/bench_result.json

@@ -1,16 +0,0 @@
-{
-    "alert": {
-        "threshold": "125%",
-        "bigger_is_better": false
-    },
-    "result_extraction": {
-        "search_pattern": "Semaphore latency: +[0-9.]+",
-        "result_index": 3
-    },
-    "chart": {
-        "title": "[Semaphores] The cost of semop",
-        "description": "lat_sem",
-        "unit": "\u00b5s",
-        "legend": "Average semaphore latency on {system}"
-    }
-}

test/benchmark/lmbench/semaphore_lat/bench_result.yaml

@@ -0,0 +1,11 @@
+alert:
+  bigger_is_better: false
+  threshold: 125%
+chart:
+  description: lat_sem
+  legend: Average semaphore latency on {system}
+  title: '[Semaphores] The cost of semop'
+  unit: "\xB5s"   # Unicode for microsecond
+result_extraction:
+  result_index: 3
+  search_pattern: 'Semaphore latency: +[0-9.]+'

test/benchmark/lmbench/signal_catch_lat/bench_result.json

@@ -1,16 +0,0 @@
-{
-    "alert": {
-        "threshold": "125%",
-        "bigger_is_better": false
-    },
-    "result_extraction": {
-        "search_pattern": "Signal handler overhead: +[0-9.]+",
-        "result_index": 4
-    },
-    "chart": {
-        "title": "[Signals] The cost of catching a signal",
-        "description": "lat_sig catch",
-        "unit": "\u00b5s",
-        "legend": "Average Signal handler overhead on {system}"
-    }
-}

test/benchmark/lmbench/signal_catch_lat/bench_result.yaml

@@ -0,0 +1,11 @@
+alert:
+  bigger_is_better: false
+  threshold: 125%
+chart:
+  description: lat_sig catch
+  legend: Average Signal handler overhead on {system}
+  title: '[Signals] The cost of catching a signal'
+  unit: "\xB5s"   # Unicode for microsecond
+result_extraction:
+  result_index: 4
+  search_pattern: 'Signal handler overhead: +[0-9.]+'

test/benchmark/lmbench/signal_install_lat/bench_result.json

@@ -1,16 +0,0 @@
-{
-    "alert": {
-        "threshold": "125%",
-        "bigger_is_better": false
-    },
-    "result_extraction": {
-        "search_pattern": "Signal handler installation: +[0-9.]+",
-        "result_index": 4
-    },
-    "chart": {
-        "title": "[Signals] The cost of installing a signal handler",
-        "description": "lat_sig install",
-        "unit": "\u00b5s",
-        "legend": "Average Signal handler install latency on {system}"
-    }
-}

test/benchmark/lmbench/signal_install_lat/bench_result.yaml

@@ -0,0 +1,11 @@
+alert:
+  bigger_is_better: false
+  threshold: 125%
+chart:
+  description: lat_sig install
+  legend: Average Signal handler install latency on {system}
+  title: '[Signals] The cost of installing a signal handler'
+  unit: "\xB5s"   # Unicode for microsecond
+result_extraction:
+  result_index: 4
+  search_pattern: 'Signal handler installation: +[0-9.]+'

test/benchmark/lmbench/signal_prot_lat/bench_result.json

@@ -1,16 +0,0 @@
-{
-    "alert": {
-        "threshold": "125%",
-        "bigger_is_better": false
-    },
-    "result_extraction": {
-        "search_pattern": "Protection fault: +[0-9.]+",
-        "result_index": 3
-    },
-    "chart": {
-        "title": "[Signals] The cost of catching a segfault",
-        "description": "lat_sig prot",
-        "unit": "\u00b5s",
-        "legend": "Average protection fault latency on {system}"
-    }
-}

test/benchmark/lmbench/signal_prot_lat/bench_result.yaml

@@ -0,0 +1,11 @@
+alert:
+  bigger_is_better: false
+  threshold: 125%
+chart:
+  description: lat_sig prot
+  legend: Average protection fault latency on {system}
+  title: '[Signals] The cost of catching a segfault'
+  unit: "\xB5s"   # Unicode for microsecond
+result_extraction:
+  result_index: 3
+  search_pattern: 'Protection fault: +[0-9.]+'

test/benchmark/lmbench/summary.json

@@ -1,44 +0,0 @@
-{
-    "benchmarks": [
-        "mem_pagefault_lat",
-        "mem_mmap_bw",
-        "mem_mmap_lat",
-        "process_getppid_lat",
-        "process_ctx_lat",
-        "process_fork_lat",
-        "process_exec_lat",
-        "process_shell_lat",
-        "signal_install_lat",
-        "signal_catch_lat",
-        "signal_prot_lat",
-        "pipe_lat",
-        "pipe_bw",
-        "fifo_lat",
-        "unix_lat",
-        "unix_bw",
-        "unix_connect_lat",
-        "vfs_open_lat",
-        "vfs_read_lat",
-        "vfs_write_lat",
-        "vfs_stat_lat",
-        "vfs_fstat_lat",
-        "vfs_read_pagecache_bw",
-        "vfs_select_lat",
-        "ramfs_copy_files_bw",
-        "ramfs_create_delete_files_0k_ops",
-        "ramfs_create_delete_files_10k_ops",
-        "ext2_copy_files_bw",
-        "tcp_loopback_bw_128",
-        "tcp_loopback_bw_4k",
-        "tcp_loopback_bw_64k",
-        "tcp_loopback_lat",
-        "tcp_loopback_select_lat",
-        "tcp_loopback_http_bw",
-        "udp_loopback_lat",
-        "tcp_virtio_bw_128",
-        "tcp_virtio_bw_64k",
-        "tcp_virtio_connect_lat",
-        "tcp_virtio_lat",
-        "udp_virtio_lat"
-    ]
-}

test/benchmark/lmbench/summary.yaml

@@ -0,0 +1,45 @@
+benchmarks:
+  - mem_pagefault_lat
+  - mem_mmap_bw
+  - mem_mmap_lat
+  - process_getppid_lat
+  - process_ctx_lat
+  - process_fork_lat
+  - process_exec_lat
+  - process_shell_lat
+  - signal_install_lat
+  - signal_catch_lat
+  - signal_prot_lat
+  - pipe_lat
+  - pipe_bw
+  - fifo_lat
+  - unix_lat
+  - unix_bw
+  - unix_connect_lat
+  - vfs_open_lat
+  - vfs_read_lat
+  - vfs_write_lat
+  - vfs_stat_lat
+  - vfs_fstat_lat
+  - vfs_read_pagecache_bw
+  - vfs_select_lat
+  - ramfs_copy_files_bw
+  - ramfs_create_delete_files_0k_ops
+  - ramfs_create_delete_files_10k_ops
+  - ext2_copy_files_bw
+  - tcp_loopback_bw_128
+  - tcp_loopback_bw_4k
+  - tcp_loopback_bw_64k
+  - tcp_loopback_lat
+  - tcp_loopback_select_lat
+  - tcp_loopback_http_bw
+  - udp_loopback_lat
+  - tcp_virtio_bw_128
+  - tcp_virtio_bw_64k
+  - tcp_virtio_connect_lat
+  - tcp_virtio_lat
+  - udp_virtio_lat
+  - # mem_copy_bw
+  - # mem_read_bw
+  - # mem_write_bw
+  - # vfs_fcntl_lat

test/benchmark/lmbench/tcp_loopback_bw_128/bench_result.json

@@ -1,16 +0,0 @@
-{
-    "alert": {
-        "threshold": "125%",
-        "bigger_is_better": true
-    },
-    "result_extraction": {
-        "search_pattern": "0.000128 +[0-9.]+",
-        "result_index": 2
-    },
-    "chart": {
-        "title": "[TCP sockets] The bandwidth (loopback, 128B message)",
-        "description": "bw_tcp -l",
-        "unit": "MB/s",
-        "legend": "Average TCP bandwidth on {system}"
-    }
-}

test/benchmark/lmbench/tcp_loopback_bw_128/bench_result.yaml

@@ -0,0 +1,11 @@
+alert:
+  bigger_is_better: true
+  threshold: 125%
+chart:
+  description: bw_tcp -l
+  legend: Average TCP bandwidth on {system}
+  title: '[TCP sockets] The bandwidth (loopback, 128B message)'
+  unit: MB/s
+result_extraction:
+  result_index: 2
+  search_pattern: 0.000128 +[0-9.]+

test/benchmark/lmbench/tcp_loopback_bw_4k/bench_result.json

@@ -1,16 +0,0 @@
-{
-    "alert": {
-        "threshold": "125%",
-        "bigger_is_better": true
-    },
-    "result_extraction": {
-        "search_pattern": "0.004096 +[0-9.]+",
-        "result_index": 2
-    },
-    "chart": {
-        "title": "[TCP sockets] The bandwidth (loopback, 4KB message)",
-        "description": "bw_tcp -l",
-        "unit": "MB/s",
-        "legend": "Average TCP bandwidth on {system}"
-    }
-}

test/benchmark/lmbench/tcp_loopback_bw_4k/bench_result.yaml

@@ -0,0 +1,11 @@
+alert:
+  bigger_is_better: true
+  threshold: 125%
+chart:
+  description: bw_tcp -l
+  legend: Average TCP bandwidth on {system}
+  title: '[TCP sockets] The bandwidth (loopback, 4KB message)'
+  unit: MB/s
+result_extraction:
+  result_index: 2
+  search_pattern: 0.004096 +[0-9.]+

test/benchmark/lmbench/tcp_loopback_bw_64k/bench_result.json

@@ -1,16 +0,0 @@
-{
-    "alert": {
-        "threshold": "125%",
-        "bigger_is_better": true
-    },
-    "result_extraction": {
-        "search_pattern": "0.065536 +[0-9.]+ +MB",
-        "result_index": 2
-    },
-    "chart": {
-        "title": "[TCP sockets] The bandwidth (loopback, 64KB message)",
-        "description": "bw_tcp -l",
-        "unit": "MB/s",
-        "legend": "Average TCP bandwidth on {system}"
-    }
-}

test/benchmark/lmbench/tcp_loopback_bw_64k/bench_result.yaml

@@ -0,0 +1,11 @@
+alert:
+  bigger_is_better: true
+  threshold: 125%
+chart:
+  description: bw_tcp -l
+  legend: Average TCP bandwidth on {system}
+  title: '[TCP sockets] The bandwidth (loopback, 64KB message)'
+  unit: MB/s
+result_extraction:
+  result_index: 2
+  search_pattern: 0.065536 +[0-9.]+ +MB

test/benchmark/lmbench/tcp_loopback_connect_lat/bench_result.json

@@ -1,16 +0,0 @@
-{
-    "alert": {
-        "threshold": "125%",
-        "bigger_is_better": false
-    },
-    "result_extraction": {
-        "search_pattern": "TCP\\/IP connection cost to [0-9.]+: +[0-9.]+",
-        "result_index": 6
-    },
-    "chart": {
-        "title": "[TCP sockets] The latency of connect (loopback)",
-        "description": "lat_connect",
-        "unit": "\u00b5s",
-        "legend": "Average TCP connection latency on {system}"
-    }
-}

test/benchmark/lmbench/tcp_loopback_connect_lat/bench_result.yaml

@@ -0,0 +1,11 @@
+alert:
+  bigger_is_better: false
+  threshold: 125%
+chart:
+  description: lat_connect
+  legend: Average TCP connection latency on {system}
+  title: '[TCP sockets] The latency of connect (loopback)'
+  unit: "\xB5s"   # Unicode for microsecond
+result_extraction:
+  result_index: 6
+  search_pattern: 'TCP\/IP connection cost to [0-9.]+: +[0-9.]+'

test/benchmark/lmbench/tcp_loopback_http_bw/bench_result.json

@@ -1,16 +0,0 @@
-{
-    "alert": {
-        "threshold": "125%",
-        "bigger_is_better": true
-    },
-    "result_extraction": {
-        "search_pattern": "Avg xfer: +([0-9.]+|inf)",
-        "result_index": 8
-    },
-    "chart": {
-        "title": "[HTTP] The bandwidth (loopback)",
-        "description": "bw_http",
-        "unit": "MB/s",
-        "legend": "Average simple HTTP transaction bandwidth on {system}"
-    }
-}

test/benchmark/lmbench/tcp_loopback_http_bw/bench_result.yaml

@@ -0,0 +1,11 @@
+alert:
+  bigger_is_better: true
+  threshold: 125%
+chart:
+  description: bw_http
+  legend: Average simple HTTP transaction bandwidth on {system}
+  title: '[HTTP] The bandwidth (loopback)'
+  unit: MB/s
+result_extraction:
+  result_index: 8
+  search_pattern: 'Avg xfer: +([0-9.]+|inf)'

test/benchmark/lmbench/tcp_loopback_lat/bench_result.json

@@ -1,16 +0,0 @@
-{
-    "alert": {
-        "threshold": "125%",
-        "bigger_is_better": false
-    },
-    "result_extraction": {
-        "search_pattern": "TCP latency using [0-9.]+: +[0-9.]+",
-        "result_index": 5
-    },
-    "chart": {
-        "title": "[TCP sockets] The latency of write+read (loopback, 1B message)",
-        "description": "lat_tcp",
-        "unit": "\u00b5s",
-        "legend": "Average TCP latency on {system}"
-    }
-}

test/benchmark/lmbench/tcp_loopback_lat/bench_result.yaml

@@ -0,0 +1,11 @@
+alert:
+  bigger_is_better: false
+  threshold: 125%
+chart:
+  description: lat_tcp
+  legend: Average TCP latency on {system}
+  title: '[TCP sockets] The latency of write+read (loopback, 1B message)'
+  unit: "\xB5s"   # Unicode for microsecond
+result_extraction:
+  result_index: 5
+  search_pattern: 'TCP latency using [0-9.]+: +[0-9.]+'

test/benchmark/lmbench/tcp_loopback_select_lat/bench_result.json

@@ -1,16 +0,0 @@
-{
-    "alert": {
-        "threshold": "125%",
-        "bigger_is_better": false
-    },
-    "result_extraction": {
-        "search_pattern": "Select on [0-9.]+ tcp fd's: +[0-9.]+",
-        "result_index": 6
-    },
-    "chart": {
-        "title": "[Network] The cost of select (TCP fds)",
-        "description": "lat_select",
-        "unit": "\u00b5s",
-        "legend": "Average select TCP latency on {system}"
-    }
-}

test/benchmark/lmbench/tcp_loopback_select_lat/bench_result.yaml

@@ -0,0 +1,11 @@
+alert:
+  bigger_is_better: false
+  threshold: 125%
+chart:
+  description: lat_select
+  legend: Average select TCP latency on {system}
+  title: '[Network] The cost of select (TCP fds)'
+  unit: "\xB5s"   # Unicode for microsecond
+result_extraction:
+  result_index: 6
+  search_pattern: 'Select on [0-9.]+ tcp fd''s: +[0-9.]+'

test/benchmark/lmbench/tcp_virtio_bw_128/bench_result.json

@@ -1,16 +0,0 @@
-{
-    "alert": {
-        "threshold": "125%",
-        "bigger_is_better": true
-    },
-    "result_extraction": {
-        "search_pattern": "0.000128 +[0-9.]+ +MB",
-        "result_index": 2
-    },
-    "chart": {
-        "title": "[TCP sockets] The bandwidth (virtio-net, 128B message)",
-        "description": "bw_tcp -l",
-        "unit": "MB/sec",
-        "legend": "Average TCP bandwidth on {system}"
-    }
-}

test/benchmark/lmbench/tcp_virtio_bw_128/bench_result.yaml

@@ -0,0 +1,11 @@
+alert:
+  bigger_is_better: true
+  threshold: 125%
+chart:
+  description: bw_tcp -l
+  legend: Average TCP bandwidth on {system}
+  title: '[TCP sockets] The bandwidth (virtio-net, 128B message)'
+  unit: MB/sec
+result_extraction:
+  result_index: 2
+  search_pattern: 0.000128 +[0-9.]+ +MB

test/benchmark/lmbench/tcp_virtio_bw_64k/bench_result.json

@@ -1,16 +0,0 @@
-{
-    "alert": {
-        "threshold": "125%",
-        "bigger_is_better": true
-    },
-    "result_extraction": {
-        "search_pattern": "0.065536 +[0-9.]+ +MB",
-        "result_index": 2
-    },
-    "chart": {
-        "title": "[TCP sockets] The bandwidth (virtio-net, 64KB message)",
-        "description": "bw_tcp -l",
-        "unit": "MB/sec",
-        "legend": "Average TCP bandwidth on {system}"
-    }
-}

test/benchmark/lmbench/tcp_virtio_bw_64k/bench_result.yaml

@@ -0,0 +1,11 @@
+alert:
+  bigger_is_better: true
+  threshold: 125%
+chart:
+  description: bw_tcp -l
+  legend: Average TCP bandwidth on {system}
+  title: '[TCP sockets] The bandwidth (virtio-net, 64KB message)'
+  unit: MB/sec
+result_extraction:
+  result_index: 2
+  search_pattern: 0.065536 +[0-9.]+ +MB

test/benchmark/lmbench/tcp_virtio_connect_lat/bench_result.json

@@ -1,16 +0,0 @@
-{
-    "alert": {
-        "threshold": "125%",
-        "bigger_is_better": false
-    },
-    "result_extraction": {
-        "search_pattern": "TCP\\/IP connection cost to [0-9.]+: +[0-9.]+",
-        "result_index": 6
-    },
-    "chart": {
-        "title": "[TCP sockets] The latency of connect (virtio-net)",
-        "description": "lat_connect",
-        "unit": "\u00b5s",
-        "legend": "Average TCP connection latency on {system}"
-    }
-}

test/benchmark/lmbench/tcp_virtio_connect_lat/bench_result.yaml

@@ -0,0 +1,11 @@
+alert:
+  bigger_is_better: false
+  threshold: 125%
+chart:
+  description: lat_connect
+  legend: Average TCP connection latency on {system}
+  title: '[TCP sockets] The latency of connect (virtio-net)'
+  unit: "\xB5s"   # Unicode for microsecond
+result_extraction:
+  result_index: 6
+  search_pattern: 'TCP\/IP connection cost to [0-9.]+: +[0-9.]+'

test/benchmark/lmbench/tcp_virtio_lat/bench_result.json

@@ -1,16 +0,0 @@
-{
-    "alert": {
-        "threshold": "125%",
-        "bigger_is_better": false
-    },
-    "result_extraction": {
-        "search_pattern": "TCP latency using [0-9.]+: +[0-9.]+",
-        "result_index": 5
-    },
-    "chart": {
-        "title": "[TCP sockets] The latency of write+read (virtio-net, 1B message)",
-        "description": "lat_tcp_virtio",
-        "unit": "\u00b5s",
-        "legend": "Average TCP latency over virtio-net on {system}"
-    }
-}

test/benchmark/lmbench/tcp_virtio_lat/bench_result.yaml

@@ -0,0 +1,11 @@
+alert:
+  bigger_is_better: false
+  threshold: 125%
+chart:
+  description: lat_tcp_virtio
+  legend: Average TCP latency over virtio-net on {system}
+  title: '[TCP sockets] The latency of write+read (virtio-net, 1B message)'
+  unit: "\xB5s"   # Unicode for microsecond
+result_extraction:
+  result_index: 5
+  search_pattern: 'TCP latency using [0-9.]+: +[0-9.]+'

test/benchmark/lmbench/udp_loopback_lat/bench_result.json

@@ -1,16 +0,0 @@
-{
-    "alert": {
-        "threshold": "125%",
-        "bigger_is_better": false
-    },
-    "result_extraction": {
-        "search_pattern": "UDP latency using [0-9.]+: +[0-9.]+",
-        "result_index": 5
-    },
-    "chart": {
-        "title": "[UDP sockets] The latency of write+read (loopback, 4B message)",
-        "description": "lat_udp",
-        "unit": "\u00b5s",
-        "legend": "Average UDP latency on {system}"
-    }
-}

test/benchmark/lmbench/udp_loopback_lat/bench_result.yaml

@@ -0,0 +1,11 @@
+alert:
+  bigger_is_better: false
+  threshold: 125%
+chart:
+  description: lat_udp
+  legend: Average UDP latency on {system}
+  title: '[UDP sockets] The latency of write+read (loopback, 4B message)'
+  unit: "\xB5s"   # Unicode for microsecond
+result_extraction:
+  result_index: 5
+  search_pattern: 'UDP latency using [0-9.]+: +[0-9.]+'

test/benchmark/lmbench/udp_virtio_lat/bench_result.json

@@ -1,16 +0,0 @@
-{
-    "alert": {
-        "threshold": "125%",
-        "bigger_is_better": false
-    },
-    "result_extraction": {
-        "search_pattern": "UDP latency using [0-9.]+: +[0-9.]+",
-        "result_index": 5
-    },
-    "chart": {
-        "title": "[UDP sockets] The latency of write+read (virtio-net, 4B message)",
-        "description": "lat_udp",
-        "unit": "\u00b5s",
-        "legend": "Average UDP latency over virtio-net on {system}"
-    }
-}