mirror of
https://github.com/asterinas/asterinas.git
synced 2025-06-08 21:06:48 +00:00
Submit all benchmark results in one step
This commit is contained in:
Fabing Li
Tate, Hongliang Tian
parent
600f13ef78
commit
d7fbdbfc63
149
.github/workflows/benchmark_x86.yml
vendored
149
.github/workflows/benchmark_x86.yml
vendored
@ -109,11 +109,6 @@ jobs:
|
||||
container:
|
||||
image: asterinas/asterinas:0.14.1-20250326
|
||||
options: --device=/dev/kvm --privileged
|
||||
env:
|
||||
# Need to set up proxy since the self-hosted CI server is located in China,
|
||||
# which has poor network connection to the official Rust crate repositories.
|
||||
RUSTUP_DIST_SERVER: https://mirrors.ustc.edu.cn/rust-static
|
||||
RUSTUP_UPDATE_ROOT: https://mirrors.ustc.edu.cn/rust-static/rustup
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
@ -126,15 +121,11 @@ jobs:
|
||||
git config --global http.version HTTP/1.1
|
||||
|
||||
- name: Run benchmarks
|
||||
uses: nick-invision/retry@v3 # Retry the benchmark command in case of failure
|
||||
with:
|
||||
timeout_minutes: 20
|
||||
max_attempts: 3
|
||||
command: |
|
||||
make install_osdk
|
||||
bash test/benchmark/bench_linux_and_aster.sh "${{ matrix.benchmarks }}"
|
||||
BENCHMARK_ARTIFACT=results_$(echo "${{ matrix.benchmarks }}" | tr '/' '-')
|
||||
echo "BENCHMARK_ARTIFACT=$BENCHMARK_ARTIFACT" >> $GITHUB_ENV
|
||||
run: |
|
||||
make install_osdk
|
||||
bash test/benchmark/bench_linux_and_aster.sh "${{ matrix.benchmarks }}"
|
||||
BENCHMARK_ARTIFACT=results_$(echo "${{ matrix.benchmarks }}" | tr '/' '-')
|
||||
echo "BENCHMARK_ARTIFACT=$BENCHMARK_ARTIFACT" >> $GITHUB_ENV
|
||||
|
||||
- name: Store benchmark results
|
||||
uses: actions/upload-artifact@v4
|
||||
@ -144,35 +135,10 @@ jobs:
|
||||
path: |
|
||||
result_*.json
|
||||
|
||||
Matrix:
|
||||
runs-on: ubuntu-latest
|
||||
needs: Benchmarks # Must run after the Benchmarks job.
|
||||
if: always() # Always run regardless of whether the previous job was successful or not.
|
||||
outputs:
|
||||
benchmarks: ${{ steps.set-matrix.outputs.benchmarks }}
|
||||
steps:
|
||||
- name: Download results
|
||||
uses: actions/download-artifact@v4
|
||||
with:
|
||||
pattern: results_*
|
||||
path: ./results
|
||||
merge-multiple: true
|
||||
|
||||
- name: Set matrix for benchmark results
|
||||
id: set-matrix
|
||||
run: |
|
||||
benchmarks=$(ls results/result_*.json | sed 's/.*result_//' | sed 's/\.json//' | jq -R -s -c 'split("\n")[:-1]')
|
||||
echo benchmarks=$benchmarks >> $GITHUB_OUTPUT
|
||||
|
||||
Results:
|
||||
runs-on: ubuntu-latest
|
||||
needs: Matrix
|
||||
if: always() # Always run regardless of whether the previous job was successful or not.
|
||||
strategy:
|
||||
matrix:
|
||||
benchmark: ${{ fromJson(needs.Matrix.outputs.benchmarks) }}
|
||||
max-parallel: 1
|
||||
fail-fast: false
|
||||
needs: Benchmarks
|
||||
if: always()
|
||||
|
||||
steps:
|
||||
- uses: actions/checkout@v4
|
||||
@ -182,44 +148,89 @@ jobs:
|
||||
pattern: results_*
|
||||
path: ./results
|
||||
merge-multiple: true
|
||||
|
||||
|
||||
- name: Set up the environment
|
||||
run: |
|
||||
sudo apt-get update && sudo apt-get install -y yq
|
||||
sudo apt-get update && sudo apt-get install -y yq jq
|
||||
|
||||
- name: Set up benchmark configuration
|
||||
- name: Generate all benchmark config files
|
||||
run: |
|
||||
BENCHMARK_DIR=$(echo ${{ matrix.benchmark }} | sed 's/-/\//g')
|
||||
BENCHMARK_SUITE=$(echo $BENCHMARK_DIR | awk -F'/' '{print $1}')
|
||||
BENCHMARK_NAME=$(echo "$BENCHMARK_DIR" | sed -E 's|^[^/]+/||; s|/bench_results||g; s|/|_|g')
|
||||
BENCH_RESULT="test/benchmark/${BENCHMARK_DIR}/bench_result.yaml"
|
||||
[ -f "$BENCH_RESULT" ] || BENCH_RESULT="test/benchmark/${BENCHMARK_DIR}.yaml"
|
||||
ALERT_THRESHOLD=$(yq -r '.alert.threshold // "130%"' "$BENCH_RESULT")
|
||||
ALERT_TOOL=$(yq -r 'if (.alert.bigger_is_better == true) then "customBiggerIsBetter" else "customSmallerIsBetter" end' "$BENCH_RESULT")
|
||||
TITLE=$(yq -r '.chart.title // "Undefined"' "$BENCH_RESULT")
|
||||
DESCRIPTION=$(yq -r '.chart.description // "No description provided"' "$BENCH_RESULT")
|
||||
echo "BENCHMARK_SUITE=$BENCHMARK_SUITE" >> $GITHUB_ENV
|
||||
echo "BENCHMARK_NAME=$BENCHMARK_NAME" >> $GITHUB_ENV
|
||||
echo "ALERT_THRESHOLD=$ALERT_THRESHOLD" >> $GITHUB_ENV
|
||||
echo "ALERT_TOOL=$ALERT_TOOL" >> $GITHUB_ENV
|
||||
echo "TITLE=$TITLE" >> $GITHUB_ENV
|
||||
echo "DESCRIPTION=$DESCRIPTION" >> $GITHUB_ENV
|
||||
yq . test/benchmark/$BENCHMARK_SUITE/summary.yaml > test/benchmark/$BENCHMARK_SUITE/summary.json
|
||||
mkdir -p configs
|
||||
BENCHMARK_LIST=$(ls results/result_*.json | sed 's/.*result_//' | sed 's/\.json//' | jq -R -s -c 'split("\n")[:-1]')
|
||||
echo "Processing benchmarks: $BENCHMARK_LIST"
|
||||
|
||||
# Loop through the benchmark identifiers provided by the Matrix job
|
||||
for benchmark_id in $(echo "$BENCHMARK_LIST" | jq -r '.[]'); do
|
||||
echo "--- Processing $benchmark_id ---"
|
||||
BENCHMARK_DIR=$(echo "$benchmark_id" | sed 's/-/\//g')
|
||||
BENCHMARK_SUITE=$(echo "$BENCHMARK_DIR" | awk -F'/' '{print $1}')
|
||||
BENCHMARK_NAME=$(echo "$BENCHMARK_DIR" | sed -E 's|^[^/]+/||; s|/bench_results||g; s|/|_|g')
|
||||
BENCH_RESULT_YAML="test/benchmark/${BENCHMARK_DIR}/bench_result.yaml"
|
||||
[ -f "$BENCH_RESULT_YAML" ] || BENCH_RESULT_YAML="test/benchmark/${BENCHMARK_DIR}.yaml"
|
||||
|
||||
if [ ! -f "$BENCH_RESULT_YAML" ]; then
|
||||
echo "Warning: YAML file not found for $benchmark_id at $BENCH_RESULT_YAML. Skipping config generation."
|
||||
continue
|
||||
fi
|
||||
|
||||
# Extract data using yq
|
||||
ALERT_THRESHOLD=$(yq -r '.alert.threshold // "130%"' "$BENCH_RESULT_YAML")
|
||||
ALERT_TOOL=$(yq -r 'if (.alert.bigger_is_better == true) then "customBiggerIsBetter" else "customSmallerIsBetter" end' "$BENCH_RESULT_YAML")
|
||||
TITLE=$(yq -r '.chart.title // "Undefined"' "$BENCH_RESULT_YAML")
|
||||
DESCRIPTION=$(yq -r '.chart.description // "No description provided"' "$BENCH_RESULT_YAML")
|
||||
|
||||
# Generate summary JSON if needed (only once per suite)
|
||||
SUMMARY_JSON="test/benchmark/$BENCHMARK_SUITE/summary.json"
|
||||
if [ ! -f "$SUMMARY_JSON" ]; then
|
||||
SUMMARY_YAML="test/benchmark/$BENCHMARK_SUITE/summary.yaml"
|
||||
if [ -f "$SUMMARY_YAML" ]; then
|
||||
yq . "$SUMMARY_YAML" > "$SUMMARY_JSON"
|
||||
echo "Generated $SUMMARY_JSON"
|
||||
else
|
||||
echo "Warning: summary.yaml not found for suite $BENCHMARK_SUITE"
|
||||
fi
|
||||
fi
|
||||
|
||||
# Define file paths
|
||||
CONFIG_FILE="configs/config_${benchmark_id}.json"
|
||||
RESULT_FILE="results/result_${benchmark_id}.json"
|
||||
|
||||
# Create JSON structure using jq
|
||||
jq -n \
|
||||
--arg title "$TITLE" \
|
||||
--arg description "$DESCRIPTION" \
|
||||
--arg suite "$BENCHMARK_SUITE" \
|
||||
--arg name "$BENCHMARK_NAME" \
|
||||
--arg threshold "$ALERT_THRESHOLD" \
|
||||
--arg tool "$ALERT_TOOL" \
|
||||
--arg result_path "$RESULT_FILE" \
|
||||
--arg summary_path "$SUMMARY_JSON" \
|
||||
'{
|
||||
metadata: {
|
||||
title: $title,
|
||||
description: $description,
|
||||
suite: $suite,
|
||||
name: $name,
|
||||
threshold: $threshold,
|
||||
tool: $tool,
|
||||
summary: $summary_path
|
||||
},
|
||||
result: $result_path
|
||||
}' > "$CONFIG_FILE"
|
||||
|
||||
echo "Generated config file $CONFIG_FILE"
|
||||
done
|
||||
|
||||
- name: Store benchmark results
|
||||
uses: asterinas/github-action-benchmark@v4
|
||||
uses: asterinas/github-action-benchmark@v5
|
||||
with:
|
||||
name: ${{ env.BENCHMARK_NAME }}
|
||||
tool: ${{ env.ALERT_TOOL }}
|
||||
output-file-path: results/result_${{ matrix.benchmark }}.json
|
||||
benchmark-data-dir-path: ${{ env.BENCHMARK_SUITE }}
|
||||
# Use glob pattern to find all generated config files
|
||||
output-file-path: "configs/config_*.json"
|
||||
github-token: ${{ secrets.BENCHMARK_SECRET }}
|
||||
gh-repository: 'github.com/asterinas/benchmark'
|
||||
auto-push: true
|
||||
alert-threshold: ${{ env.ALERT_THRESHOLD }}
|
||||
comment-on-alert: true
|
||||
fail-on-alert: false
|
||||
chart-title: ${{ env.TITLE }}
|
||||
chart-description: ${{ env.DESCRIPTION }}
|
||||
summary-json-path: test/benchmark/${{ env.BENCHMARK_SUITE }}/summary.json
|
||||
max-items-in-chart: 60
|
||||
ref: ${{ github.sha }}
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user