From d7fbdbfc637e22d776f522779d85d5ae0b47c376 Mon Sep 17 00:00:00 2001 From: Fabing Li Date: Thu, 3 Apr 2025 02:22:15 +0000 Subject: [PATCH] Submit all benchmark results in one step --- .github/workflows/benchmark_x86.yml | 149 +++++++++++++++------------- 1 file changed, 80 insertions(+), 69 deletions(-) diff --git a/.github/workflows/benchmark_x86.yml b/.github/workflows/benchmark_x86.yml index 401df5ddc..707e34f8c 100644 --- a/.github/workflows/benchmark_x86.yml +++ b/.github/workflows/benchmark_x86.yml @@ -109,11 +109,6 @@ jobs: container: image: asterinas/asterinas:0.14.1-20250326 options: --device=/dev/kvm --privileged - env: - # Need to set up proxy since the self-hosted CI server is located in China, - # which has poor network connection to the official Rust crate repositories. - RUSTUP_DIST_SERVER: https://mirrors.ustc.edu.cn/rust-static - RUSTUP_UPDATE_ROOT: https://mirrors.ustc.edu.cn/rust-static/rustup steps: - uses: actions/checkout@v4 @@ -126,15 +121,11 @@ jobs: git config --global http.version HTTP/1.1 - name: Run benchmarks - uses: nick-invision/retry@v3 # Retry the benchmark command in case of failure - with: - timeout_minutes: 20 - max_attempts: 3 - command: | - make install_osdk - bash test/benchmark/bench_linux_and_aster.sh "${{ matrix.benchmarks }}" - BENCHMARK_ARTIFACT=results_$(echo "${{ matrix.benchmarks }}" | tr '/' '-') - echo "BENCHMARK_ARTIFACT=$BENCHMARK_ARTIFACT" >> $GITHUB_ENV + run: | + make install_osdk + bash test/benchmark/bench_linux_and_aster.sh "${{ matrix.benchmarks }}" + BENCHMARK_ARTIFACT=results_$(echo "${{ matrix.benchmarks }}" | tr '/' '-') + echo "BENCHMARK_ARTIFACT=$BENCHMARK_ARTIFACT" >> $GITHUB_ENV - name: Store benchmark results uses: actions/upload-artifact@v4 @@ -144,35 +135,10 @@ jobs: path: | result_*.json - Matrix: - runs-on: ubuntu-latest - needs: Benchmarks # Must run after the Benchmarks job. - if: always() # Always run regardless of whether the previous job was successful or not. - outputs: - benchmarks: ${{ steps.set-matrix.outputs.benchmarks }} - steps: - - name: Download results - uses: actions/download-artifact@v4 - with: - pattern: results_* - path: ./results - merge-multiple: true - - - name: Set matrix for benchmark results - id: set-matrix - run: | - benchmarks=$(ls results/result_*.json | sed 's/.*result_//' | sed 's/\.json//' | jq -R -s -c 'split("\n")[:-1]') - echo benchmarks=$benchmarks >> $GITHUB_OUTPUT - Results: runs-on: ubuntu-latest - needs: Matrix - if: always() # Always run regardless of whether the previous job was successful or not. - strategy: - matrix: - benchmark: ${{ fromJson(needs.Matrix.outputs.benchmarks) }} - max-parallel: 1 - fail-fast: false + needs: Benchmarks + if: always() steps: - uses: actions/checkout@v4 @@ -182,44 +148,89 @@ jobs: pattern: results_* path: ./results merge-multiple: true - + - name: Set up the environment run: | - sudo apt-get update && sudo apt-get install -y yq + sudo apt-get update && sudo apt-get install -y yq jq - - name: Set up benchmark configuration + - name: Generate all benchmark config files run: | - BENCHMARK_DIR=$(echo ${{ matrix.benchmark }} | sed 's/-/\//g') - BENCHMARK_SUITE=$(echo $BENCHMARK_DIR | awk -F'/' '{print $1}') - BENCHMARK_NAME=$(echo "$BENCHMARK_DIR" | sed -E 's|^[^/]+/||; s|/bench_results||g; s|/|_|g') - BENCH_RESULT="test/benchmark/${BENCHMARK_DIR}/bench_result.yaml" - [ -f "$BENCH_RESULT" ] || BENCH_RESULT="test/benchmark/${BENCHMARK_DIR}.yaml" - ALERT_THRESHOLD=$(yq -r '.alert.threshold // "130%"' "$BENCH_RESULT") - ALERT_TOOL=$(yq -r 'if (.alert.bigger_is_better == true) then "customBiggerIsBetter" else "customSmallerIsBetter" end' "$BENCH_RESULT") - TITLE=$(yq -r '.chart.title // "Undefined"' "$BENCH_RESULT") - DESCRIPTION=$(yq -r '.chart.description // "No description provided"' "$BENCH_RESULT") - echo "BENCHMARK_SUITE=$BENCHMARK_SUITE" >> $GITHUB_ENV - echo "BENCHMARK_NAME=$BENCHMARK_NAME" >> $GITHUB_ENV - echo "ALERT_THRESHOLD=$ALERT_THRESHOLD" >> $GITHUB_ENV - echo "ALERT_TOOL=$ALERT_TOOL" >> $GITHUB_ENV - echo "TITLE=$TITLE" >> $GITHUB_ENV - echo "DESCRIPTION=$DESCRIPTION" >> $GITHUB_ENV - yq . test/benchmark/$BENCHMARK_SUITE/summary.yaml > test/benchmark/$BENCHMARK_SUITE/summary.json + mkdir -p configs + BENCHMARK_LIST=$(ls results/result_*.json | sed 's/.*result_//' | sed 's/\.json//' | jq -R -s -c 'split("\n")[:-1]') + echo "Processing benchmarks: $BENCHMARK_LIST" + + # Loop through the benchmark identifiers provided by the Matrix job + for benchmark_id in $(echo "$BENCHMARK_LIST" | jq -r '.[]'); do + echo "--- Processing $benchmark_id ---" + BENCHMARK_DIR=$(echo "$benchmark_id" | sed 's/-/\//g') + BENCHMARK_SUITE=$(echo "$BENCHMARK_DIR" | awk -F'/' '{print $1}') + BENCHMARK_NAME=$(echo "$BENCHMARK_DIR" | sed -E 's|^[^/]+/||; s|/bench_results||g; s|/|_|g') + BENCH_RESULT_YAML="test/benchmark/${BENCHMARK_DIR}/bench_result.yaml" + [ -f "$BENCH_RESULT_YAML" ] || BENCH_RESULT_YAML="test/benchmark/${BENCHMARK_DIR}.yaml" + + if [ ! -f "$BENCH_RESULT_YAML" ]; then + echo "Warning: YAML file not found for $benchmark_id at $BENCH_RESULT_YAML. Skipping config generation." + continue + fi + + # Extract data using yq + ALERT_THRESHOLD=$(yq -r '.alert.threshold // "130%"' "$BENCH_RESULT_YAML") + ALERT_TOOL=$(yq -r 'if (.alert.bigger_is_better == true) then "customBiggerIsBetter" else "customSmallerIsBetter" end' "$BENCH_RESULT_YAML") + TITLE=$(yq -r '.chart.title // "Undefined"' "$BENCH_RESULT_YAML") + DESCRIPTION=$(yq -r '.chart.description // "No description provided"' "$BENCH_RESULT_YAML") + + # Generate summary JSON if needed (only once per suite) + SUMMARY_JSON="test/benchmark/$BENCHMARK_SUITE/summary.json" + if [ ! -f "$SUMMARY_JSON" ]; then + SUMMARY_YAML="test/benchmark/$BENCHMARK_SUITE/summary.yaml" + if [ -f "$SUMMARY_YAML" ]; then + yq . "$SUMMARY_YAML" > "$SUMMARY_JSON" + echo "Generated $SUMMARY_JSON" + else + echo "Warning: summary.yaml not found for suite $BENCHMARK_SUITE" + fi + fi + + # Define file paths + CONFIG_FILE="configs/config_${benchmark_id}.json" + RESULT_FILE="results/result_${benchmark_id}.json" + + # Create JSON structure using jq + jq -n \ + --arg title "$TITLE" \ + --arg description "$DESCRIPTION" \ + --arg suite "$BENCHMARK_SUITE" \ + --arg name "$BENCHMARK_NAME" \ + --arg threshold "$ALERT_THRESHOLD" \ + --arg tool "$ALERT_TOOL" \ + --arg result_path "$RESULT_FILE" \ + --arg summary_path "$SUMMARY_JSON" \ + '{ + metadata: { + title: $title, + description: $description, + suite: $suite, + name: $name, + threshold: $threshold, + tool: $tool, + summary: $summary_path + }, + result: $result_path + }' > "$CONFIG_FILE" + + echo "Generated config file $CONFIG_FILE" + done - name: Store benchmark results - uses: asterinas/github-action-benchmark@v4 + uses: asterinas/github-action-benchmark@v5 with: - name: ${{ env.BENCHMARK_NAME }} - tool: ${{ env.ALERT_TOOL }} - output-file-path: results/result_${{ matrix.benchmark }}.json - benchmark-data-dir-path: ${{ env.BENCHMARK_SUITE }} + # Use glob pattern to find all generated config files + output-file-path: "configs/config_*.json" github-token: ${{ secrets.BENCHMARK_SECRET }} gh-repository: 'github.com/asterinas/benchmark' auto-push: true - alert-threshold: ${{ env.ALERT_THRESHOLD }} comment-on-alert: true fail-on-alert: false - chart-title: ${{ env.TITLE }} - chart-description: ${{ env.DESCRIPTION }} - summary-json-path: test/benchmark/${{ env.BENCHMARK_SUITE }}/summary.json + max-items-in-chart: 60 ref: ${{ github.sha }} + \ No newline at end of file