Skip to content

Commit 6cbba5b

Browse files
pablogsalpablogsal
committed
Add JIT to daily runs and reschedule the upload
1 parent 71dd85d commit 6cbba5b

File tree

5 files changed

+548
-28
lines changed

5 files changed

+548
-28
lines changed

‎.claude/settings.local.json‎

Lines changed: 0 additions & 22 deletions
This file was deleted.

‎.github/workflows/daily-benchmark-debug.yml‎

Lines changed: 175 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,175 @@
1+
name: Daily Memory Tracker Benchmark - Debug Build
2+
3+
on:
4+
schedule:
5+
# Run daily at 23:00 UTC (EOD) to pick up all commits from the day
6+
- cron: '0 23 * * *'
7+
workflow_dispatch:
8+
inputs:
9+
target_date:
10+
description: 'Date to get commits from (YYYY-MM-DD, defaults to today)'
11+
required: false
12+
type: string
13+
binary_id:
14+
description: 'Binary ID to use for benchmarking'
15+
required: false
16+
default: 'debug'
17+
environment_id:
18+
description: 'Environment ID'
19+
required: false
20+
default: 'gh_actions'
21+
server_url:
22+
description: 'Memory tracker server URL'
23+
required: false
24+
default: 'https://memory.python.org'
25+
cpython_repo:
26+
description: 'CPython repository URL'
27+
required: false
28+
default: 'https://github.com/python/cpython.git'
29+
30+
jobs:
31+
get-daily-commits:
32+
runs-on: ubuntu-latest
33+
outputs:
34+
commits: ${{ steps.get-commits.outputs.commits }}
35+
commit-count: ${{ steps.get-commits.outputs.commit-count }}
36+
37+
steps:
38+
- name: Clone CPython repository
39+
run: |
40+
git clone ${{ github.event.inputs.cpython_repo || 'https://github.com/python/cpython.git' }} cpython
41+
cd cpython
42+
git fetch --all
43+
44+
- name: Get commits from target date
45+
id: get-commits
46+
run: |
47+
cd cpython
48+
49+
# Determine target date
50+
if [ -n "${{ github.event.inputs.target_date }}" ]; then
51+
TARGET_DATE="${{ github.event.inputs.target_date }}"
52+
else
53+
TARGET_DATE=$(date -u +%Y-%m-%d)
54+
fi
55+
56+
echo "Getting commits from date: $TARGET_DATE"
57+
58+
# Get commits from the target date (00:00 to 23:59 UTC)
59+
COMMITS=$(git log --since="$TARGET_DATE 00:00:00 UTC" --until="$TARGET_DATE 23:59:59 UTC" --pretty=format:"%H" --reverse)
60+
61+
if [ -z "$COMMITS" ]; then
62+
echo "No commits found for date $TARGET_DATE"
63+
echo "commits=" >> $GITHUB_OUTPUT
64+
echo "commit-count=0" >> $GITHUB_OUTPUT
65+
else
66+
# Convert to JSON array format for matrix strategy
67+
COMMITS_JSON=$(echo "$COMMITS" | jq -R -s -c 'split("\n") | map(select(length > 0))')
68+
COMMIT_COUNT=$(echo "$COMMITS" | wc -l)
69+
70+
echo "Found $COMMIT_COUNT commits for date $TARGET_DATE"
71+
echo "commits=$COMMITS_JSON" >> $GITHUB_OUTPUT
72+
echo "commit-count=$COMMIT_COUNT" >> $GITHUB_OUTPUT
73+
74+
echo "Commits to benchmark:"
75+
echo "$COMMITS"
76+
fi
77+
78+
benchmark-commits:
79+
needs: get-daily-commits
80+
if: needs.get-daily-commits.outputs.commit-count > 0
81+
runs-on: ubuntu-latest
82+
strategy:
83+
matrix:
84+
commit: ${{ fromJson(needs.get-daily-commits.outputs.commits) }}
85+
fail-fast: false
86+
max-parallel: 5
87+
88+
steps:
89+
- name: Checkout memory tracker
90+
uses: actions/checkout@v4
91+
92+
- name: Set up Python
93+
uses: actions/setup-python@v4
94+
with:
95+
python-version: '3.11'
96+
97+
- name: Clone CPython repository
98+
run: |
99+
git clone ${{ github.event.inputs.cpython_repo || 'https://github.com/python/cpython.git' }} cpython
100+
cd cpython
101+
git fetch --depth=200
102+
103+
- name: Install memory tracker worker
104+
run: |
105+
cd worker
106+
pip install -e .
107+
108+
- name: Install build dependencies
109+
run: |
110+
# Install CPython dependencies using their script
111+
cd cpython
112+
sudo .github/workflows/posix-deps-apt.sh
113+
114+
# Install Memray dependencies
115+
sudo apt-get install -y \
116+
python3-dev \
117+
libdebuginfod-dev \
118+
libunwind-dev \
119+
liblz4-dev
120+
121+
- name: Run memory benchmark for commit
122+
env:
123+
MEMORY_TRACKER_TOKEN: ${{ secrets.MEMORY_TRACKER_TOKEN }}
124+
run: |
125+
COMMIT="${{ matrix.commit }}"
126+
127+
# Build command for single commit
128+
CMD="memory-tracker benchmark '$COMMIT'"
129+
CMD="$CMD --repo-path ./cpython"
130+
CMD="$CMD --binary-id '${{ github.event.inputs.binary_id || 'debug' }}'"
131+
CMD="$CMD --environment-id '${{ github.event.inputs.environment_id || 'gh_actions' }}'"
132+
CMD="$CMD --api-base '${{ github.event.inputs.server_url || 'https://memory.python.org' }}'"
133+
CMD="$CMD --output-dir ./benchmark_results"
134+
CMD="$CMD --configure-flags='--with-pydebug'"
135+
CMD="$CMD --force"
136+
CMD="$CMD -vv"
137+
138+
echo "Running benchmark for commit: $COMMIT"
139+
echo "Command: $CMD"
140+
eval $CMD
141+
142+
- name: Upload benchmark results (if failed)
143+
if: failure()
144+
uses: actions/upload-artifact@v4
145+
with:
146+
name: benchmark-logs-${{ matrix.commit }}
147+
path: |
148+
*.log
149+
./benchmark_results/
150+
retention-days: 7
151+
152+
- name: Upload benchmark results (on success)
153+
if: success()
154+
uses: actions/upload-artifact@v4
155+
with:
156+
name: benchmark-results-${{ matrix.commit }}
157+
path: ./benchmark_results/
158+
retention-days: 30
159+
160+
summary:
161+
needs: [get-daily-commits, benchmark-commits]
162+
if: always()
163+
runs-on: ubuntu-latest
164+
165+
steps:
166+
- name: Print summary
167+
run: |
168+
echo "Daily benchmark run completed"
169+
echo "Total commits processed: ${{ needs.get-daily-commits.outputs.commit-count }}"
170+
171+
if [ "${{ needs.get-daily-commits.outputs.commit-count }}" = "0" ]; then
172+
echo "No commits found for the target date"
173+
else
174+
echo "Benchmark jobs completed with status: ${{ needs.benchmark-commits.result }}"
175+
fi

‎.github/workflows/daily-benchmark-freethreaded.yml‎

Lines changed: 175 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,175 @@
1+
name: Daily Memory Tracker Benchmark - Free-threaded Build
2+
3+
on:
4+
schedule:
5+
# Run daily at 23:00 UTC (EOD) to pick up all commits from the day
6+
- cron: '0 23 * * *'
7+
workflow_dispatch:
8+
inputs:
9+
target_date:
10+
description: 'Date to get commits from (YYYY-MM-DD, defaults to today)'
11+
required: false
12+
type: string
13+
binary_id:
14+
description: 'Binary ID to use for benchmarking'
15+
required: false
16+
default: 'nogil'
17+
environment_id:
18+
description: 'Environment ID'
19+
required: false
20+
default: 'gh_actions'
21+
server_url:
22+
description: 'Memory tracker server URL'
23+
required: false
24+
default: 'https://memory.python.org'
25+
cpython_repo:
26+
description: 'CPython repository URL'
27+
required: false
28+
default: 'https://github.com/python/cpython.git'
29+
30+
jobs:
31+
get-daily-commits:
32+
runs-on: ubuntu-latest
33+
outputs:
34+
commits: ${{ steps.get-commits.outputs.commits }}
35+
commit-count: ${{ steps.get-commits.outputs.commit-count }}
36+
37+
steps:
38+
- name: Clone CPython repository
39+
run: |
40+
git clone ${{ github.event.inputs.cpython_repo || 'https://github.com/python/cpython.git' }} cpython
41+
cd cpython
42+
git fetch --all
43+
44+
- name: Get commits from target date
45+
id: get-commits
46+
run: |
47+
cd cpython
48+
49+
# Determine target date
50+
if [ -n "${{ github.event.inputs.target_date }}" ]; then
51+
TARGET_DATE="${{ github.event.inputs.target_date }}"
52+
else
53+
TARGET_DATE=$(date -u +%Y-%m-%d)
54+
fi
55+
56+
echo "Getting commits from date: $TARGET_DATE"
57+
58+
# Get commits from the target date (00:00 to 23:59 UTC)
59+
COMMITS=$(git log --since="$TARGET_DATE 00:00:00 UTC" --until="$TARGET_DATE 23:59:59 UTC" --pretty=format:"%H" --reverse)
60+
61+
if [ -z "$COMMITS" ]; then
62+
echo "No commits found for date $TARGET_DATE"
63+
echo "commits=" >> $GITHUB_OUTPUT
64+
echo "commit-count=0" >> $GITHUB_OUTPUT
65+
else
66+
# Convert to JSON array format for matrix strategy
67+
COMMITS_JSON=$(echo "$COMMITS" | jq -R -s -c 'split("\n") | map(select(length > 0))')
68+
COMMIT_COUNT=$(echo "$COMMITS" | wc -l)
69+
70+
echo "Found $COMMIT_COUNT commits for date $TARGET_DATE"
71+
echo "commits=$COMMITS_JSON" >> $GITHUB_OUTPUT
72+
echo "commit-count=$COMMIT_COUNT" >> $GITHUB_OUTPUT
73+
74+
echo "Commits to benchmark:"
75+
echo "$COMMITS"
76+
fi
77+
78+
benchmark-commits:
79+
needs: get-daily-commits
80+
if: needs.get-daily-commits.outputs.commit-count > 0
81+
runs-on: ubuntu-latest
82+
strategy:
83+
matrix:
84+
commit: ${{ fromJson(needs.get-daily-commits.outputs.commits) }}
85+
fail-fast: false
86+
max-parallel: 5
87+
88+
steps:
89+
- name: Checkout memory tracker
90+
uses: actions/checkout@v4
91+
92+
- name: Set up Python
93+
uses: actions/setup-python@v4
94+
with:
95+
python-version: '3.11'
96+
97+
- name: Clone CPython repository
98+
run: |
99+
git clone ${{ github.event.inputs.cpython_repo || 'https://github.com/python/cpython.git' }} cpython
100+
cd cpython
101+
git fetch --depth=200
102+
103+
- name: Install memory tracker worker
104+
run: |
105+
cd worker
106+
pip install -e .
107+
108+
- name: Install build dependencies
109+
run: |
110+
# Install CPython dependencies using their script
111+
cd cpython
112+
sudo .github/workflows/posix-deps-apt.sh
113+
114+
# Install Memray dependencies
115+
sudo apt-get install -y \
116+
python3-dev \
117+
libdebuginfod-dev \
118+
libunwind-dev \
119+
liblz4-dev
120+
121+
- name: Run memory benchmark for commit
122+
env:
123+
MEMORY_TRACKER_TOKEN: ${{ secrets.MEMORY_TRACKER_TOKEN }}
124+
run: |
125+
COMMIT="${{ matrix.commit }}"
126+
127+
# Build command for single commit
128+
CMD="memory-tracker benchmark '$COMMIT'"
129+
CMD="$CMD --repo-path ./cpython"
130+
CMD="$CMD --binary-id '${{ github.event.inputs.binary_id || 'nogil' }}'"
131+
CMD="$CMD --environment-id '${{ github.event.inputs.environment_id || 'gh_actions' }}'"
132+
CMD="$CMD --api-base '${{ github.event.inputs.server_url || 'https://memory.python.org' }}'"
133+
CMD="$CMD --output-dir ./benchmark_results"
134+
CMD="$CMD --configure-flags='--disable-gil'"
135+
CMD="$CMD --force"
136+
CMD="$CMD -vv"
137+
138+
echo "Running benchmark for commit: $COMMIT"
139+
echo "Command: $CMD"
140+
eval $CMD
141+
142+
- name: Upload benchmark results (if failed)
143+
if: failure()
144+
uses: actions/upload-artifact@v4
145+
with:
146+
name: benchmark-logs-${{ matrix.commit }}
147+
path: |
148+
*.log
149+
./benchmark_results/
150+
retention-days: 7
151+
152+
- name: Upload benchmark results (on success)
153+
if: success()
154+
uses: actions/upload-artifact@v4
155+
with:
156+
name: benchmark-results-${{ matrix.commit }}
157+
path: ./benchmark_results/
158+
retention-days: 30
159+
160+
summary:
161+
needs: [get-daily-commits, benchmark-commits]
162+
if: always()
163+
runs-on: ubuntu-latest
164+
165+
steps:
166+
- name: Print summary
167+
run: |
168+
echo "Daily benchmark run completed"
169+
echo "Total commits processed: ${{ needs.get-daily-commits.outputs.commit-count }}"
170+
171+
if [ "${{ needs.get-daily-commits.outputs.commit-count }}" = "0" ]; then
172+
echo "No commits found for the target date"
173+
else
174+
echo "Benchmark jobs completed with status: ${{ needs.benchmark-commits.result }}"
175+
fi

0 commit comments

Comments
 (0)