Submit all benchmark results in one step

.github/workflows/benchmark_x86.yml

@@ -109,11 +109,6 @@ jobs:
     container: 
       image: asterinas/asterinas:0.14.1-20250326
       options: --device=/dev/kvm --privileged
-    env:
-      # Need to set up proxy since the self-hosted CI server is located in China,
-      # which has poor network connection to the official Rust crate repositories.
-      RUSTUP_DIST_SERVER: https://mirrors.ustc.edu.cn/rust-static
-      RUSTUP_UPDATE_ROOT: https://mirrors.ustc.edu.cn/rust-static/rustup
 
     steps:
       - uses: actions/checkout@v4
@@ -126,15 +121,11 @@ jobs:
           git config --global http.version HTTP/1.1
 
       - name: Run benchmarks
-        uses: nick-invision/retry@v3 # Retry the benchmark command in case of failure
+        run: |
-        with:
+          make install_osdk
-          timeout_minutes: 20
+          bash test/benchmark/bench_linux_and_aster.sh "${{ matrix.benchmarks }}"
-          max_attempts: 3
+          BENCHMARK_ARTIFACT=results_$(echo "${{ matrix.benchmarks }}" | tr '/' '-')
-          command: |
+          echo "BENCHMARK_ARTIFACT=$BENCHMARK_ARTIFACT" >> $GITHUB_ENV
-            make install_osdk
-            bash test/benchmark/bench_linux_and_aster.sh "${{ matrix.benchmarks }}"
-            BENCHMARK_ARTIFACT=results_$(echo "${{ matrix.benchmarks }}" | tr '/' '-')
-            echo "BENCHMARK_ARTIFACT=$BENCHMARK_ARTIFACT" >> $GITHUB_ENV
 
       - name: Store benchmark results
         uses: actions/upload-artifact@v4
@@ -144,35 +135,10 @@ jobs:
           path: |
             result_*.json
 
-  Matrix:
-    runs-on: ubuntu-latest
-    needs: Benchmarks # Must run after the Benchmarks job.
-    if: always() # Always run regardless of whether the previous job was successful or not.
-    outputs:
-      benchmarks: ${{ steps.set-matrix.outputs.benchmarks }}
-    steps:
-      - name: Download results
-        uses: actions/download-artifact@v4
-        with:
-          pattern: results_*
-          path: ./results
-          merge-multiple: true
-
-      - name: Set matrix for benchmark results
-        id: set-matrix
-        run: |
-          benchmarks=$(ls results/result_*.json | sed 's/.*result_//' | sed 's/\.json//' | jq -R -s -c 'split("\n")[:-1]')
-          echo benchmarks=$benchmarks >> $GITHUB_OUTPUT
-
   Results:
     runs-on: ubuntu-latest
-    needs: Matrix
+    needs: Benchmarks
-    if: always() # Always run regardless of whether the previous job was successful or not.
+    if: always()
-    strategy:
-      matrix:
-        benchmark: ${{ fromJson(needs.Matrix.outputs.benchmarks) }}
-      max-parallel: 1
-      fail-fast: false
 
     steps:
       - uses: actions/checkout@v4
@@ -185,41 +151,86 @@ jobs:
 
       - name: Set up the environment
         run: |
-          sudo apt-get update && sudo apt-get install -y yq
+          sudo apt-get update && sudo apt-get install -y yq jq
 
-      - name: Set up benchmark configuration
+      - name: Generate all benchmark config files
         run: |
-          BENCHMARK_DIR=$(echo ${{ matrix.benchmark }} | sed 's/-/\//g')
+          mkdir -p configs
-          BENCHMARK_SUITE=$(echo $BENCHMARK_DIR | awk -F'/' '{print $1}')
+          BENCHMARK_LIST=$(ls results/result_*.json | sed 's/.*result_//' | sed 's/\.json//' | jq -R -s -c 'split("\n")[:-1]')
-          BENCHMARK_NAME=$(echo "$BENCHMARK_DIR" | sed -E 's|^[^/]+/||; s|/bench_results||g; s|/|_|g')
+          echo "Processing benchmarks: $BENCHMARK_LIST"
-          BENCH_RESULT="test/benchmark/${BENCHMARK_DIR}/bench_result.yaml"
+
-          [ -f "$BENCH_RESULT" ] || BENCH_RESULT="test/benchmark/${BENCHMARK_DIR}.yaml"
+          # Loop through the benchmark identifiers provided by the Matrix job
-          ALERT_THRESHOLD=$(yq -r '.alert.threshold // "130%"' "$BENCH_RESULT")
+          for benchmark_id in $(echo "$BENCHMARK_LIST" | jq -r '.[]'); do
-          ALERT_TOOL=$(yq -r 'if (.alert.bigger_is_better == true) then "customBiggerIsBetter" else "customSmallerIsBetter" end' "$BENCH_RESULT")
+            echo "--- Processing $benchmark_id ---"
-          TITLE=$(yq -r '.chart.title // "Undefined"' "$BENCH_RESULT")
+            BENCHMARK_DIR=$(echo "$benchmark_id" | sed 's/-/\//g')
-          DESCRIPTION=$(yq -r '.chart.description // "No description provided"' "$BENCH_RESULT")
+            BENCHMARK_SUITE=$(echo "$BENCHMARK_DIR" | awk -F'/' '{print $1}')
-          echo "BENCHMARK_SUITE=$BENCHMARK_SUITE" >> $GITHUB_ENV
+            BENCHMARK_NAME=$(echo "$BENCHMARK_DIR" | sed -E 's|^[^/]+/||; s|/bench_results||g; s|/|_|g')
-          echo "BENCHMARK_NAME=$BENCHMARK_NAME" >> $GITHUB_ENV
+            BENCH_RESULT_YAML="test/benchmark/${BENCHMARK_DIR}/bench_result.yaml"
-          echo "ALERT_THRESHOLD=$ALERT_THRESHOLD" >> $GITHUB_ENV
+            [ -f "$BENCH_RESULT_YAML" ] || BENCH_RESULT_YAML="test/benchmark/${BENCHMARK_DIR}.yaml"
-          echo "ALERT_TOOL=$ALERT_TOOL" >> $GITHUB_ENV
+
-          echo "TITLE=$TITLE" >> $GITHUB_ENV
+            if [ ! -f "$BENCH_RESULT_YAML" ]; then
-          echo "DESCRIPTION=$DESCRIPTION" >> $GITHUB_ENV
+              echo "Warning: YAML file not found for $benchmark_id at $BENCH_RESULT_YAML. Skipping config generation."
-          yq . test/benchmark/$BENCHMARK_SUITE/summary.yaml > test/benchmark/$BENCHMARK_SUITE/summary.json
+              continue
+            fi
+
+            # Extract data using yq
+            ALERT_THRESHOLD=$(yq -r '.alert.threshold // "130%"' "$BENCH_RESULT_YAML")
+            ALERT_TOOL=$(yq -r 'if (.alert.bigger_is_better == true) then "customBiggerIsBetter" else "customSmallerIsBetter" end' "$BENCH_RESULT_YAML")
+            TITLE=$(yq -r '.chart.title // "Undefined"' "$BENCH_RESULT_YAML")
+            DESCRIPTION=$(yq -r '.chart.description // "No description provided"' "$BENCH_RESULT_YAML")
+
+            # Generate summary JSON if needed (only once per suite)
+            SUMMARY_JSON="test/benchmark/$BENCHMARK_SUITE/summary.json"
+            if [ ! -f "$SUMMARY_JSON" ]; then
+               SUMMARY_YAML="test/benchmark/$BENCHMARK_SUITE/summary.yaml"
+               if [ -f "$SUMMARY_YAML" ]; then
+                 yq . "$SUMMARY_YAML" > "$SUMMARY_JSON"
+                 echo "Generated $SUMMARY_JSON"
+               else
+                 echo "Warning: summary.yaml not found for suite $BENCHMARK_SUITE"
+               fi
+            fi
+
+            # Define file paths
+            CONFIG_FILE="configs/config_${benchmark_id}.json"
+            RESULT_FILE="results/result_${benchmark_id}.json"
+          
+            # Create JSON structure using jq
+            jq -n \
+              --arg title "$TITLE" \
+              --arg description "$DESCRIPTION" \
+              --arg suite "$BENCHMARK_SUITE" \
+              --arg name "$BENCHMARK_NAME" \
+              --arg threshold "$ALERT_THRESHOLD" \
+              --arg tool "$ALERT_TOOL" \
+              --arg result_path "$RESULT_FILE" \
+              --arg summary_path "$SUMMARY_JSON" \
+              '{
+                metadata: {
+                  title: $title,
+                  description: $description,
+                  suite: $suite,
+                  name: $name,
+                  threshold: $threshold,
+                  tool: $tool,
+                  summary: $summary_path
+                },
+                result: $result_path
+              }' > "$CONFIG_FILE"
+
+            echo "Generated config file $CONFIG_FILE"
+          done
 
       - name: Store benchmark results
-        uses: asterinas/github-action-benchmark@v4
+        uses: asterinas/github-action-benchmark@v5
         with:
-          name: ${{ env.BENCHMARK_NAME }}
+          # Use glob pattern to find all generated config files
-          tool: ${{ env.ALERT_TOOL }}
+          output-file-path: "configs/config_*.json"
-          output-file-path: results/result_${{ matrix.benchmark }}.json
-          benchmark-data-dir-path: ${{ env.BENCHMARK_SUITE }}
           github-token: ${{ secrets.BENCHMARK_SECRET }}
           gh-repository: 'github.com/asterinas/benchmark'
           auto-push: true
-          alert-threshold: ${{ env.ALERT_THRESHOLD }}
           comment-on-alert: true
           fail-on-alert: false
-          chart-title: ${{ env.TITLE }}
+          max-items-in-chart: 60
-          chart-description: ${{ env.DESCRIPTION }}
-          summary-json-path: test/benchmark/${{ env.BENCHMARK_SUITE }}/summary.json
           ref: ${{ github.sha }}
+