zhimin-z
commited on
Commit
·
f51e3c7
1
Parent(s):
1bae49b
add
Browse files
README.md
CHANGED
|
@@ -28,6 +28,8 @@ If an assistant can consistently resolve issues and discussions across different
|
|
| 28 |
Key metrics from the last 180 days:
|
| 29 |
|
| 30 |
**Leaderboard Table**
|
|
|
|
|
|
|
| 31 |
- **Issue Resolved Rate (%)**: Percentage of closed issues successfully resolved
|
| 32 |
- **Discussion Resolved Rate (%)**: Percentage of discussions successfully resolved (answered or closed)
|
| 33 |
- **Total Issues**: Issues the assistant has been involved with (authored, assigned, or commented on)
|
|
@@ -51,7 +53,7 @@ We focus on 180 days to highlight current capabilities and active assistants.
|
|
| 51 |
**Data Collection**
|
| 52 |
We mine GitHub activity from [GHArchive](https://www.gharchive.org/), tracking three types of activities:
|
| 53 |
|
| 54 |
-
1. **
|
| 55 |
- Issues opened or assigned to the assistant (`IssuesEvent`)
|
| 56 |
- Issue comments by the assistant (`IssueCommentEvent`)
|
| 57 |
|
|
@@ -78,7 +80,7 @@ Anyone can submit an assistant. We store metadata in `SWE-Arena/bot_metadata` an
|
|
| 78 |
- Searchable table (by assistant name or website)
|
| 79 |
- Filterable columns (by issue resolved rate, discussion resolved rate)
|
| 80 |
- Monthly charts (issue and discussion resolution trends and activity)
|
| 81 |
-
- View
|
| 82 |
|
| 83 |
**Issues Wanted Tab**:
|
| 84 |
- Browse long-standing open issues (30+ days) from major open-source projects
|
|
|
|
| 28 |
Key metrics from the last 180 days:
|
| 29 |
|
| 30 |
**Leaderboard Table**
|
| 31 |
+
- **Assistant**: Display name of the assistant
|
| 32 |
+
- **Website**: Link to the assistant's homepage or documentation
|
| 33 |
- **Issue Resolved Rate (%)**: Percentage of closed issues successfully resolved
|
| 34 |
- **Discussion Resolved Rate (%)**: Percentage of discussions successfully resolved (answered or closed)
|
| 35 |
- **Total Issues**: Issues the assistant has been involved with (authored, assigned, or commented on)
|
|
|
|
| 53 |
**Data Collection**
|
| 54 |
We mine GitHub activity from [GHArchive](https://www.gharchive.org/), tracking three types of activities:
|
| 55 |
|
| 56 |
+
1. **Assistant-Assigned Issues**:
|
| 57 |
- Issues opened or assigned to the assistant (`IssuesEvent`)
|
| 58 |
- Issue comments by the assistant (`IssueCommentEvent`)
|
| 59 |
|
|
|
|
| 80 |
- Searchable table (by assistant name or website)
|
| 81 |
- Filterable columns (by issue resolved rate, discussion resolved rate)
|
| 82 |
- Monthly charts (issue and discussion resolution trends and activity)
|
| 83 |
+
- View assistant-assigned metrics, wanted issue resolutions, and discussion metrics
|
| 84 |
|
| 85 |
**Issues Wanted Tab**:
|
| 86 |
- Browse long-standing open issues (30+ days) from major open-source projects
|
app.py
CHANGED
|
@@ -24,7 +24,7 @@ load_dotenv()
|
|
| 24 |
# CONFIGURATION
|
| 25 |
# =============================================================================
|
| 26 |
|
| 27 |
-
AGENTS_REPO = "SWE-Arena/bot_metadata" # HuggingFace dataset for
|
| 28 |
AGENTS_REPO_LOCAL_PATH = os.path.expanduser("~/bot_metadata") # Local git clone path
|
| 29 |
LEADERBOARD_FILENAME = f"{os.getenv('COMPOSE_PROJECT_NAME')}.json"
|
| 30 |
LEADERBOARD_REPO = "SWE-Arena/leaderboard_data" # HuggingFace dataset for leaderboard data
|
|
@@ -33,7 +33,7 @@ GIT_SYNC_TIMEOUT = 300 # 5 minutes timeout for git pull
|
|
| 33 |
MAX_RETRIES = 5
|
| 34 |
|
| 35 |
LEADERBOARD_COLUMNS = [
|
| 36 |
-
("
|
| 37 |
("Website", "string"),
|
| 38 |
("Total Issues", "number"),
|
| 39 |
("Total Discussions", "number"),
|
|
@@ -162,14 +162,14 @@ def sync_agents_repo():
|
|
| 162 |
|
| 163 |
def load_agents_from_hf():
|
| 164 |
"""
|
| 165 |
-
Load all
|
| 166 |
ALWAYS syncs with remote first to ensure we have the latest bot data.
|
| 167 |
"""
|
| 168 |
# MANDATORY: Sync with remote first to get latest bot data
|
| 169 |
-
print(f" Syncing bot_metadata repository to get latest
|
| 170 |
sync_agents_repo() # Will raise exception if sync fails
|
| 171 |
|
| 172 |
-
|
| 173 |
|
| 174 |
# Scan local directory for JSON files
|
| 175 |
if not os.path.exists(AGENTS_REPO_LOCAL_PATH):
|
|
@@ -177,7 +177,7 @@ def load_agents_from_hf():
|
|
| 177 |
|
| 178 |
# Walk through the directory to find all JSON files
|
| 179 |
files_processed = 0
|
| 180 |
-
print(f" Loading
|
| 181 |
|
| 182 |
for root, dirs, files in os.walk(AGENTS_REPO_LOCAL_PATH):
|
| 183 |
# Skip .git directory
|
|
@@ -195,7 +195,7 @@ def load_agents_from_hf():
|
|
| 195 |
with open(file_path, 'r', encoding='utf-8') as f:
|
| 196 |
agent_data = json.load(f)
|
| 197 |
|
| 198 |
-
# Only include active
|
| 199 |
if agent_data.get('status') != 'active':
|
| 200 |
continue
|
| 201 |
|
|
@@ -203,14 +203,14 @@ def load_agents_from_hf():
|
|
| 203 |
github_identifier = filename.replace('.json', '')
|
| 204 |
agent_data['github_identifier'] = github_identifier
|
| 205 |
|
| 206 |
-
|
| 207 |
|
| 208 |
except Exception as e:
|
| 209 |
print(f" Warning Error loading {filename}: {str(e)}")
|
| 210 |
continue
|
| 211 |
|
| 212 |
-
print(f" Success Loaded {len(
|
| 213 |
-
return
|
| 214 |
|
| 215 |
|
| 216 |
def get_hf_token():
|
|
@@ -265,7 +265,7 @@ def upload_with_retry(api, path_or_fileobj, path_in_repo, repo_id, repo_type, to
|
|
| 265 |
|
| 266 |
|
| 267 |
def save_agent_to_hf(data):
|
| 268 |
-
"""Save a new
|
| 269 |
try:
|
| 270 |
api = HfApi()
|
| 271 |
token = get_hf_token()
|
|
@@ -290,7 +290,7 @@ def save_agent_to_hf(data):
|
|
| 290 |
repo_type="dataset",
|
| 291 |
token=token
|
| 292 |
)
|
| 293 |
-
print(f"Saved
|
| 294 |
return True
|
| 295 |
finally:
|
| 296 |
# Always clean up local file, even if upload fails
|
|
@@ -298,7 +298,7 @@ def save_agent_to_hf(data):
|
|
| 298 |
os.remove(filename)
|
| 299 |
|
| 300 |
except Exception as e:
|
| 301 |
-
print(f"Error saving
|
| 302 |
return False
|
| 303 |
|
| 304 |
|
|
@@ -345,10 +345,10 @@ def create_monthly_metrics_plot(top_n=5):
|
|
| 345 |
- Left y-axis: Resolved Rate (%) as line curves
|
| 346 |
- Right y-axis: Total Issues created as bar charts
|
| 347 |
|
| 348 |
-
Each
|
| 349 |
|
| 350 |
Args:
|
| 351 |
-
top_n: Number of top
|
| 352 |
"""
|
| 353 |
# Load from saved dataset
|
| 354 |
saved_data = load_leaderboard_data_from_hf()
|
|
@@ -373,10 +373,10 @@ def create_monthly_metrics_plot(top_n=5):
|
|
| 373 |
print(f"Loaded monthly metrics from saved dataset")
|
| 374 |
|
| 375 |
# Apply top_n filter if specified
|
| 376 |
-
if top_n is not None and top_n > 0 and metrics.get('
|
| 377 |
-
# Calculate total issues for each
|
| 378 |
agent_totals = []
|
| 379 |
-
for agent_name in metrics['
|
| 380 |
agent_data = metrics['data'].get(agent_name, {})
|
| 381 |
total_issues = sum(agent_data.get('total_issues', []))
|
| 382 |
agent_totals.append((agent_name, total_issues))
|
|
@@ -385,14 +385,14 @@ def create_monthly_metrics_plot(top_n=5):
|
|
| 385 |
agent_totals.sort(key=lambda x: x[1], reverse=True)
|
| 386 |
top_agents = [agent_name for agent_name, _ in agent_totals[:top_n]]
|
| 387 |
|
| 388 |
-
# Filter metrics to only include top
|
| 389 |
metrics = {
|
| 390 |
-
'
|
| 391 |
'months': metrics['months'],
|
| 392 |
-
'data': {
|
| 393 |
}
|
| 394 |
|
| 395 |
-
if not metrics['
|
| 396 |
# Return an empty figure with a message
|
| 397 |
fig = go.Figure()
|
| 398 |
fig.add_annotation(
|
|
@@ -411,7 +411,7 @@ def create_monthly_metrics_plot(top_n=5):
|
|
| 411 |
# Create figure with secondary y-axis
|
| 412 |
fig = make_subplots(specs=[[{"secondary_y": True}]])
|
| 413 |
|
| 414 |
-
# Generate unique colors for many
|
| 415 |
def generate_color(index, total):
|
| 416 |
"""Generate distinct colors using HSL color space for better distribution"""
|
| 417 |
hue = (index * 360 / total) % 360
|
|
@@ -419,15 +419,15 @@ def create_monthly_metrics_plot(top_n=5):
|
|
| 419 |
lightness = 45 + (index % 2) * 10 # Vary lightness slightly
|
| 420 |
return f'hsl({hue}, {saturation}%, {lightness}%)'
|
| 421 |
|
| 422 |
-
|
| 423 |
months = metrics['months']
|
| 424 |
data = metrics['data']
|
| 425 |
|
| 426 |
-
# Generate colors for all
|
| 427 |
-
agent_colors = {
|
| 428 |
|
| 429 |
-
# Add traces for each
|
| 430 |
-
for idx, agent_name in enumerate(
|
| 431 |
color = agent_colors[agent_name]
|
| 432 |
agent_data = data[agent_name]
|
| 433 |
|
|
@@ -447,8 +447,8 @@ def create_monthly_metrics_plot(top_n=5):
|
|
| 447 |
line=dict(color=color, width=2),
|
| 448 |
marker=dict(size=8),
|
| 449 |
legendgroup=agent_name,
|
| 450 |
-
showlegend=(top_n is not None and top_n <= 10), # Show legend for top N
|
| 451 |
-
hovertemplate='<b>
|
| 452 |
'Month: %{x}<br>' +
|
| 453 |
'Resolved Rate: %{y:.2f}%<br>' +
|
| 454 |
'<extra></extra>'
|
|
@@ -457,7 +457,7 @@ def create_monthly_metrics_plot(top_n=5):
|
|
| 457 |
)
|
| 458 |
|
| 459 |
# Add bar trace for total issues (right y-axis)
|
| 460 |
-
# Only show bars for months where
|
| 461 |
x_bars = []
|
| 462 |
y_bars = []
|
| 463 |
for month, count in zip(months, agent_data['total_issues']):
|
|
@@ -474,11 +474,11 @@ def create_monthly_metrics_plot(top_n=5):
|
|
| 474 |
marker=dict(color=color, opacity=0.6),
|
| 475 |
legendgroup=agent_name,
|
| 476 |
showlegend=False, # Hide duplicate legend entry (already shown in Scatter)
|
| 477 |
-
hovertemplate='<b>
|
| 478 |
'Month: %{x}<br>' +
|
| 479 |
'Total Issues: %{y}<br>' +
|
| 480 |
'<extra></extra>',
|
| 481 |
-
offsetgroup=agent_name # Group bars by
|
| 482 |
),
|
| 483 |
secondary_y=True
|
| 484 |
)
|
|
@@ -500,7 +500,7 @@ def create_monthly_metrics_plot(top_n=5):
|
|
| 500 |
show_legend = (top_n is not None and top_n <= 10)
|
| 501 |
fig.update_layout(
|
| 502 |
title=None,
|
| 503 |
-
hovermode='closest', # Show individual
|
| 504 |
barmode='group',
|
| 505 |
height=600,
|
| 506 |
showlegend=show_legend,
|
|
@@ -516,10 +516,10 @@ def create_discussion_monthly_metrics_plot(top_n=5):
|
|
| 516 |
- Left y-axis: Discussion Resolved Rate (%) as line curves
|
| 517 |
- Right y-axis: Total Discussions created as bar charts
|
| 518 |
|
| 519 |
-
Each
|
| 520 |
|
| 521 |
Args:
|
| 522 |
-
top_n: Number of top
|
| 523 |
"""
|
| 524 |
# Load from saved dataset
|
| 525 |
saved_data = load_leaderboard_data_from_hf()
|
|
@@ -544,10 +544,10 @@ def create_discussion_monthly_metrics_plot(top_n=5):
|
|
| 544 |
print(f"Loaded discussion monthly metrics from saved dataset")
|
| 545 |
|
| 546 |
# Apply top_n filter if specified
|
| 547 |
-
if top_n is not None and top_n > 0 and metrics.get('
|
| 548 |
-
# Calculate total discussions for each
|
| 549 |
agent_totals = []
|
| 550 |
-
for agent_name in metrics['
|
| 551 |
agent_data = metrics['data'].get(agent_name, {})
|
| 552 |
total_discussions = agent_data.get('total_discussions')
|
| 553 |
agent_totals.append((agent_name, total_discussions))
|
|
@@ -556,14 +556,14 @@ def create_discussion_monthly_metrics_plot(top_n=5):
|
|
| 556 |
agent_totals.sort(key=lambda x: x[1], reverse=True)
|
| 557 |
top_agents = [agent_name for agent_name, _ in agent_totals[:top_n]]
|
| 558 |
|
| 559 |
-
# Filter metrics to only include top
|
| 560 |
metrics = {
|
| 561 |
-
'
|
| 562 |
'months': metrics['months'],
|
| 563 |
-
'data': {
|
| 564 |
}
|
| 565 |
|
| 566 |
-
if not metrics['
|
| 567 |
# Return an empty figure with a message
|
| 568 |
fig = go.Figure()
|
| 569 |
fig.add_annotation(
|
|
@@ -582,7 +582,7 @@ def create_discussion_monthly_metrics_plot(top_n=5):
|
|
| 582 |
# Create figure with secondary y-axis
|
| 583 |
fig = make_subplots(specs=[[{"secondary_y": True}]])
|
| 584 |
|
| 585 |
-
# Generate unique colors for many
|
| 586 |
def generate_color(index, total):
|
| 587 |
"""Generate distinct colors using HSL color space for better distribution"""
|
| 588 |
hue = (index * 360 / total) % 360
|
|
@@ -590,15 +590,15 @@ def create_discussion_monthly_metrics_plot(top_n=5):
|
|
| 590 |
lightness = 45 + (index % 2) * 10 # Vary lightness slightly
|
| 591 |
return f'hsl({hue}, {saturation}%, {lightness}%)'
|
| 592 |
|
| 593 |
-
|
| 594 |
months = metrics['months']
|
| 595 |
data = metrics['data']
|
| 596 |
|
| 597 |
-
# Generate colors for all
|
| 598 |
-
agent_colors = {
|
| 599 |
|
| 600 |
-
# Add traces for each
|
| 601 |
-
for idx, agent_name in enumerate(
|
| 602 |
color = agent_colors[agent_name]
|
| 603 |
agent_data = data[agent_name]
|
| 604 |
|
|
@@ -618,8 +618,8 @@ def create_discussion_monthly_metrics_plot(top_n=5):
|
|
| 618 |
line=dict(color=color, width=2),
|
| 619 |
marker=dict(size=8),
|
| 620 |
legendgroup=agent_name,
|
| 621 |
-
showlegend=(top_n is not None and top_n <= 10), # Show legend for top N
|
| 622 |
-
hovertemplate='<b>
|
| 623 |
'Month: %{x}<br>' +
|
| 624 |
'Discussion Resolved Rate: %{y:.2f}%<br>' +
|
| 625 |
'<extra></extra>'
|
|
@@ -628,7 +628,7 @@ def create_discussion_monthly_metrics_plot(top_n=5):
|
|
| 628 |
)
|
| 629 |
|
| 630 |
# Add bar trace for total discussions (right y-axis)
|
| 631 |
-
# Only show bars for months where
|
| 632 |
x_bars = []
|
| 633 |
y_bars = []
|
| 634 |
for month, count in zip(months, agent_data['total_discussions']):
|
|
@@ -645,11 +645,11 @@ def create_discussion_monthly_metrics_plot(top_n=5):
|
|
| 645 |
marker=dict(color=color, opacity=0.6),
|
| 646 |
legendgroup=agent_name,
|
| 647 |
showlegend=False, # Hide duplicate legend entry (already shown in Scatter)
|
| 648 |
-
hovertemplate='<b>
|
| 649 |
'Month: %{x}<br>' +
|
| 650 |
'Total Discussions: %{y}<br>' +
|
| 651 |
'<extra></extra>',
|
| 652 |
-
offsetgroup=agent_name # Group bars by
|
| 653 |
),
|
| 654 |
secondary_y=True
|
| 655 |
)
|
|
@@ -671,7 +671,7 @@ def create_discussion_monthly_metrics_plot(top_n=5):
|
|
| 671 |
show_legend = (top_n is not None and top_n <= 10)
|
| 672 |
fig.update_layout(
|
| 673 |
title=None,
|
| 674 |
-
hovermode='closest', # Show individual
|
| 675 |
barmode='group',
|
| 676 |
height=600,
|
| 677 |
showlegend=show_legend,
|
|
@@ -710,9 +710,9 @@ def get_leaderboard_dataframe():
|
|
| 710 |
filtered_count = 0
|
| 711 |
for identifier, data in cache_dict.items():
|
| 712 |
total_issues = data.get('total_issues', 0)
|
| 713 |
-
print(f"
|
| 714 |
|
| 715 |
-
# Filter out
|
| 716 |
if total_issues == 0:
|
| 717 |
filtered_count += 1
|
| 718 |
continue
|
|
@@ -730,8 +730,8 @@ def get_leaderboard_dataframe():
|
|
| 730 |
data.get('resolved_discussions', 0), # Resolved Discussions
|
| 731 |
])
|
| 732 |
|
| 733 |
-
print(f"Filtered out {filtered_count}
|
| 734 |
-
print(f"Leaderboard will show {len(rows)}
|
| 735 |
|
| 736 |
# Create DataFrame
|
| 737 |
column_names = [col[0] for col in LEADERBOARD_COLUMNS]
|
|
@@ -807,14 +807,14 @@ def get_wanted_issues_dataframe():
|
|
| 807 |
|
| 808 |
def submit_agent(identifier, agent_name, organization, website):
|
| 809 |
"""
|
| 810 |
-
Submit a new
|
| 811 |
Validates input and saves submission.
|
| 812 |
"""
|
| 813 |
# Validate required fields
|
| 814 |
if not identifier or not identifier.strip():
|
| 815 |
return "ERROR: GitHub identifier is required", gr.update()
|
| 816 |
if not agent_name or not agent_name.strip():
|
| 817 |
-
return "ERROR:
|
| 818 |
if not organization or not organization.strip():
|
| 819 |
return "ERROR: Organization name is required", gr.update()
|
| 820 |
if not website or not website.strip():
|
|
@@ -831,12 +831,12 @@ def submit_agent(identifier, agent_name, organization, website):
|
|
| 831 |
if not is_valid:
|
| 832 |
return f"ERROR: {message}", gr.update()
|
| 833 |
|
| 834 |
-
# Check for duplicates by loading
|
| 835 |
-
|
| 836 |
-
if
|
| 837 |
-
existing_names = {
|
| 838 |
if identifier in existing_names:
|
| 839 |
-
return f"WARNING:
|
| 840 |
|
| 841 |
# Create submission
|
| 842 |
submission = {
|
|
@@ -873,7 +873,7 @@ def reload_leaderboard_data():
|
|
| 873 |
if data:
|
| 874 |
print(f"Successfully reloaded leaderboard data")
|
| 875 |
print(f" Last updated: {data.get('metadata', {}).get('last_updated', 'Unknown')}")
|
| 876 |
-
print(f"
|
| 877 |
else:
|
| 878 |
print(f"No data available")
|
| 879 |
except Exception as e:
|
|
@@ -886,7 +886,7 @@ def reload_leaderboard_data():
|
|
| 886 |
# GRADIO APPLICATION
|
| 887 |
# =============================================================================
|
| 888 |
|
| 889 |
-
print(f"\nStarting SWE
|
| 890 |
print(f" Data source: {LEADERBOARD_REPO}")
|
| 891 |
print(f" Reload frequency: Daily at 12:00 AM UTC\n")
|
| 892 |
|
|
@@ -907,19 +907,19 @@ print(f"On startup: Loads cached data from HuggingFace on demand")
|
|
| 907 |
print(f"{'='*80}\n")
|
| 908 |
|
| 909 |
# Create Gradio interface
|
| 910 |
-
with gr.Blocks(title="SWE
|
| 911 |
-
gr.Markdown("# SWE
|
| 912 |
-
gr.Markdown(f"Track and compare GitHub issue and discussion resolution statistics for SWE
|
| 913 |
|
| 914 |
with gr.Tabs():
|
| 915 |
|
| 916 |
# Leaderboard Tab
|
| 917 |
with gr.Tab("Leaderboard"):
|
| 918 |
-
gr.Markdown("*Statistics are based on
|
| 919 |
leaderboard_table = Leaderboard(
|
| 920 |
value=pd.DataFrame(columns=[col[0] for col in LEADERBOARD_COLUMNS]), # Empty initially
|
| 921 |
datatype=LEADERBOARD_COLUMNS,
|
| 922 |
-
search_columns=["
|
| 923 |
filter_columns=[
|
| 924 |
ColumnFilter(
|
| 925 |
"Issue Resolved Rate (%)",
|
|
@@ -942,8 +942,8 @@ with gr.Blocks(title="SWE Agent Issue & Discussion Leaderboard", theme=gr.themes
|
|
| 942 |
# Monthly Metrics Section
|
| 943 |
gr.Markdown("---") # Divider
|
| 944 |
with gr.Group():
|
| 945 |
-
gr.Markdown("### Issue Monthly Performance - Top 5
|
| 946 |
-
gr.Markdown("*Shows issue resolution trends and volumes for the most active
|
| 947 |
monthly_metrics_plot = gr.Plot(label="Issue Monthly Metrics")
|
| 948 |
|
| 949 |
# Load monthly metrics when app starts
|
|
@@ -956,8 +956,8 @@ with gr.Blocks(title="SWE Agent Issue & Discussion Leaderboard", theme=gr.themes
|
|
| 956 |
# Discussion Monthly Metrics Section
|
| 957 |
gr.Markdown("---") # Divider
|
| 958 |
with gr.Group():
|
| 959 |
-
gr.Markdown("### Discussion Monthly Performance - Top 5
|
| 960 |
-
gr.Markdown("*Shows discussion resolution trends and volumes for the most active
|
| 961 |
discussion_metrics_plot = gr.Plot(label="Discussion Monthly Metrics")
|
| 962 |
|
| 963 |
# Load discussion monthly metrics when app starts
|
|
@@ -987,20 +987,20 @@ with gr.Blocks(title="SWE Agent Issue & Discussion Leaderboard", theme=gr.themes
|
|
| 987 |
)
|
| 988 |
|
| 989 |
|
| 990 |
-
# Submit
|
| 991 |
-
with gr.Tab("Submit Your
|
| 992 |
|
| 993 |
-
gr.Markdown("Fill in the details below to add your
|
| 994 |
|
| 995 |
with gr.Row():
|
| 996 |
with gr.Column():
|
| 997 |
github_input = gr.Textbox(
|
| 998 |
label="GitHub Identifier*",
|
| 999 |
-
placeholder="Your
|
| 1000 |
)
|
| 1001 |
name_input = gr.Textbox(
|
| 1002 |
-
label="
|
| 1003 |
-
placeholder="Your
|
| 1004 |
)
|
| 1005 |
|
| 1006 |
with gr.Column():
|
|
@@ -1010,11 +1010,11 @@ with gr.Blocks(title="SWE Agent Issue & Discussion Leaderboard", theme=gr.themes
|
|
| 1010 |
)
|
| 1011 |
website_input = gr.Textbox(
|
| 1012 |
label="Website*",
|
| 1013 |
-
placeholder="https://your-
|
| 1014 |
)
|
| 1015 |
|
| 1016 |
submit_button = gr.Button(
|
| 1017 |
-
"Submit
|
| 1018 |
variant="primary"
|
| 1019 |
)
|
| 1020 |
submission_status = gr.Textbox(
|
|
|
|
| 24 |
# CONFIGURATION
|
| 25 |
# =============================================================================
|
| 26 |
|
| 27 |
+
AGENTS_REPO = "SWE-Arena/bot_metadata" # HuggingFace dataset for assistant metadata
|
| 28 |
AGENTS_REPO_LOCAL_PATH = os.path.expanduser("~/bot_metadata") # Local git clone path
|
| 29 |
LEADERBOARD_FILENAME = f"{os.getenv('COMPOSE_PROJECT_NAME')}.json"
|
| 30 |
LEADERBOARD_REPO = "SWE-Arena/leaderboard_data" # HuggingFace dataset for leaderboard data
|
|
|
|
| 33 |
MAX_RETRIES = 5
|
| 34 |
|
| 35 |
LEADERBOARD_COLUMNS = [
|
| 36 |
+
("Assistant", "string"),
|
| 37 |
("Website", "string"),
|
| 38 |
("Total Issues", "number"),
|
| 39 |
("Total Discussions", "number"),
|
|
|
|
| 162 |
|
| 163 |
def load_agents_from_hf():
|
| 164 |
"""
|
| 165 |
+
Load all assistant metadata JSON files from local git repository.
|
| 166 |
ALWAYS syncs with remote first to ensure we have the latest bot data.
|
| 167 |
"""
|
| 168 |
# MANDATORY: Sync with remote first to get latest bot data
|
| 169 |
+
print(f" Syncing bot_metadata repository to get latest assistants...")
|
| 170 |
sync_agents_repo() # Will raise exception if sync fails
|
| 171 |
|
| 172 |
+
assistants = []
|
| 173 |
|
| 174 |
# Scan local directory for JSON files
|
| 175 |
if not os.path.exists(AGENTS_REPO_LOCAL_PATH):
|
|
|
|
| 177 |
|
| 178 |
# Walk through the directory to find all JSON files
|
| 179 |
files_processed = 0
|
| 180 |
+
print(f" Loading assistant metadata from {AGENTS_REPO_LOCAL_PATH}...")
|
| 181 |
|
| 182 |
for root, dirs, files in os.walk(AGENTS_REPO_LOCAL_PATH):
|
| 183 |
# Skip .git directory
|
|
|
|
| 195 |
with open(file_path, 'r', encoding='utf-8') as f:
|
| 196 |
agent_data = json.load(f)
|
| 197 |
|
| 198 |
+
# Only include active assistants
|
| 199 |
if agent_data.get('status') != 'active':
|
| 200 |
continue
|
| 201 |
|
|
|
|
| 203 |
github_identifier = filename.replace('.json', '')
|
| 204 |
agent_data['github_identifier'] = github_identifier
|
| 205 |
|
| 206 |
+
assistants.append(agent_data)
|
| 207 |
|
| 208 |
except Exception as e:
|
| 209 |
print(f" Warning Error loading {filename}: {str(e)}")
|
| 210 |
continue
|
| 211 |
|
| 212 |
+
print(f" Success Loaded {len(assistants)} active assistants (from {files_processed} total files)")
|
| 213 |
+
return assistants
|
| 214 |
|
| 215 |
|
| 216 |
def get_hf_token():
|
|
|
|
| 265 |
|
| 266 |
|
| 267 |
def save_agent_to_hf(data):
|
| 268 |
+
"""Save a new assistant to HuggingFace dataset as {identifier}.json in root."""
|
| 269 |
try:
|
| 270 |
api = HfApi()
|
| 271 |
token = get_hf_token()
|
|
|
|
| 290 |
repo_type="dataset",
|
| 291 |
token=token
|
| 292 |
)
|
| 293 |
+
print(f"Saved assistant to HuggingFace: {filename}")
|
| 294 |
return True
|
| 295 |
finally:
|
| 296 |
# Always clean up local file, even if upload fails
|
|
|
|
| 298 |
os.remove(filename)
|
| 299 |
|
| 300 |
except Exception as e:
|
| 301 |
+
print(f"Error saving assistant: {str(e)}")
|
| 302 |
return False
|
| 303 |
|
| 304 |
|
|
|
|
| 345 |
- Left y-axis: Resolved Rate (%) as line curves
|
| 346 |
- Right y-axis: Total Issues created as bar charts
|
| 347 |
|
| 348 |
+
Each assistant gets a unique color for both their line and bars.
|
| 349 |
|
| 350 |
Args:
|
| 351 |
+
top_n: Number of top assistants to show (default: 5)
|
| 352 |
"""
|
| 353 |
# Load from saved dataset
|
| 354 |
saved_data = load_leaderboard_data_from_hf()
|
|
|
|
| 373 |
print(f"Loaded monthly metrics from saved dataset")
|
| 374 |
|
| 375 |
# Apply top_n filter if specified
|
| 376 |
+
if top_n is not None and top_n > 0 and metrics.get('assistants'):
|
| 377 |
+
# Calculate total issues for each assistant
|
| 378 |
agent_totals = []
|
| 379 |
+
for agent_name in metrics['assistants']:
|
| 380 |
agent_data = metrics['data'].get(agent_name, {})
|
| 381 |
total_issues = sum(agent_data.get('total_issues', []))
|
| 382 |
agent_totals.append((agent_name, total_issues))
|
|
|
|
| 385 |
agent_totals.sort(key=lambda x: x[1], reverse=True)
|
| 386 |
top_agents = [agent_name for agent_name, _ in agent_totals[:top_n]]
|
| 387 |
|
| 388 |
+
# Filter metrics to only include top assistants
|
| 389 |
metrics = {
|
| 390 |
+
'assistants': top_agents,
|
| 391 |
'months': metrics['months'],
|
| 392 |
+
'data': {assistant: metrics['data'][assistant] for assistant in top_agents if assistant in metrics['data']}
|
| 393 |
}
|
| 394 |
|
| 395 |
+
if not metrics['assistants'] or not metrics['months']:
|
| 396 |
# Return an empty figure with a message
|
| 397 |
fig = go.Figure()
|
| 398 |
fig.add_annotation(
|
|
|
|
| 411 |
# Create figure with secondary y-axis
|
| 412 |
fig = make_subplots(specs=[[{"secondary_y": True}]])
|
| 413 |
|
| 414 |
+
# Generate unique colors for many assistants using HSL color space
|
| 415 |
def generate_color(index, total):
|
| 416 |
"""Generate distinct colors using HSL color space for better distribution"""
|
| 417 |
hue = (index * 360 / total) % 360
|
|
|
|
| 419 |
lightness = 45 + (index % 2) * 10 # Vary lightness slightly
|
| 420 |
return f'hsl({hue}, {saturation}%, {lightness}%)'
|
| 421 |
|
| 422 |
+
assistants = metrics['assistants']
|
| 423 |
months = metrics['months']
|
| 424 |
data = metrics['data']
|
| 425 |
|
| 426 |
+
# Generate colors for all assistants
|
| 427 |
+
agent_colors = {assistant: generate_color(idx, len(assistants)) for idx, assistant in enumerate(assistants)}
|
| 428 |
|
| 429 |
+
# Add traces for each assistant
|
| 430 |
+
for idx, agent_name in enumerate(assistants):
|
| 431 |
color = agent_colors[agent_name]
|
| 432 |
agent_data = data[agent_name]
|
| 433 |
|
|
|
|
| 447 |
line=dict(color=color, width=2),
|
| 448 |
marker=dict(size=8),
|
| 449 |
legendgroup=agent_name,
|
| 450 |
+
showlegend=(top_n is not None and top_n <= 10), # Show legend for top N assistants
|
| 451 |
+
hovertemplate='<b>Assistant: %{fullData.name}</b><br>' +
|
| 452 |
'Month: %{x}<br>' +
|
| 453 |
'Resolved Rate: %{y:.2f}%<br>' +
|
| 454 |
'<extra></extra>'
|
|
|
|
| 457 |
)
|
| 458 |
|
| 459 |
# Add bar trace for total issues (right y-axis)
|
| 460 |
+
# Only show bars for months where assistant has issues
|
| 461 |
x_bars = []
|
| 462 |
y_bars = []
|
| 463 |
for month, count in zip(months, agent_data['total_issues']):
|
|
|
|
| 474 |
marker=dict(color=color, opacity=0.6),
|
| 475 |
legendgroup=agent_name,
|
| 476 |
showlegend=False, # Hide duplicate legend entry (already shown in Scatter)
|
| 477 |
+
hovertemplate='<b>Assistant: %{fullData.name}</b><br>' +
|
| 478 |
'Month: %{x}<br>' +
|
| 479 |
'Total Issues: %{y}<br>' +
|
| 480 |
'<extra></extra>',
|
| 481 |
+
offsetgroup=agent_name # Group bars by assistant for proper spacing
|
| 482 |
),
|
| 483 |
secondary_y=True
|
| 484 |
)
|
|
|
|
| 500 |
show_legend = (top_n is not None and top_n <= 10)
|
| 501 |
fig.update_layout(
|
| 502 |
title=None,
|
| 503 |
+
hovermode='closest', # Show individual assistant info on hover
|
| 504 |
barmode='group',
|
| 505 |
height=600,
|
| 506 |
showlegend=show_legend,
|
|
|
|
| 516 |
- Left y-axis: Discussion Resolved Rate (%) as line curves
|
| 517 |
- Right y-axis: Total Discussions created as bar charts
|
| 518 |
|
| 519 |
+
Each assistant gets a unique color for both their line and bars.
|
| 520 |
|
| 521 |
Args:
|
| 522 |
+
top_n: Number of top assistants to show (default: 5)
|
| 523 |
"""
|
| 524 |
# Load from saved dataset
|
| 525 |
saved_data = load_leaderboard_data_from_hf()
|
|
|
|
| 544 |
print(f"Loaded discussion monthly metrics from saved dataset")
|
| 545 |
|
| 546 |
# Apply top_n filter if specified
|
| 547 |
+
if top_n is not None and top_n > 0 and metrics.get('assistants'):
|
| 548 |
+
# Calculate total discussions for each assistant
|
| 549 |
agent_totals = []
|
| 550 |
+
for agent_name in metrics['assistants']:
|
| 551 |
agent_data = metrics['data'].get(agent_name, {})
|
| 552 |
total_discussions = agent_data.get('total_discussions')
|
| 553 |
agent_totals.append((agent_name, total_discussions))
|
|
|
|
| 556 |
agent_totals.sort(key=lambda x: x[1], reverse=True)
|
| 557 |
top_agents = [agent_name for agent_name, _ in agent_totals[:top_n]]
|
| 558 |
|
| 559 |
+
# Filter metrics to only include top assistants
|
| 560 |
metrics = {
|
| 561 |
+
'assistants': top_agents,
|
| 562 |
'months': metrics['months'],
|
| 563 |
+
'data': {assistant: metrics['data'][assistant] for assistant in top_agents if assistant in metrics['data']}
|
| 564 |
}
|
| 565 |
|
| 566 |
+
if not metrics['assistants'] or not metrics['months']:
|
| 567 |
# Return an empty figure with a message
|
| 568 |
fig = go.Figure()
|
| 569 |
fig.add_annotation(
|
|
|
|
| 582 |
# Create figure with secondary y-axis
|
| 583 |
fig = make_subplots(specs=[[{"secondary_y": True}]])
|
| 584 |
|
| 585 |
+
# Generate unique colors for many assistants using HSL color space
|
| 586 |
def generate_color(index, total):
|
| 587 |
"""Generate distinct colors using HSL color space for better distribution"""
|
| 588 |
hue = (index * 360 / total) % 360
|
|
|
|
| 590 |
lightness = 45 + (index % 2) * 10 # Vary lightness slightly
|
| 591 |
return f'hsl({hue}, {saturation}%, {lightness}%)'
|
| 592 |
|
| 593 |
+
assistants = metrics['assistants']
|
| 594 |
months = metrics['months']
|
| 595 |
data = metrics['data']
|
| 596 |
|
| 597 |
+
# Generate colors for all assistants
|
| 598 |
+
agent_colors = {assistant: generate_color(idx, len(assistants)) for idx, assistant in enumerate(assistants)}
|
| 599 |
|
| 600 |
+
# Add traces for each assistant
|
| 601 |
+
for idx, agent_name in enumerate(assistants):
|
| 602 |
color = agent_colors[agent_name]
|
| 603 |
agent_data = data[agent_name]
|
| 604 |
|
|
|
|
| 618 |
line=dict(color=color, width=2),
|
| 619 |
marker=dict(size=8),
|
| 620 |
legendgroup=agent_name,
|
| 621 |
+
showlegend=(top_n is not None and top_n <= 10), # Show legend for top N assistants
|
| 622 |
+
hovertemplate='<b>Assistant: %{fullData.name}</b><br>' +
|
| 623 |
'Month: %{x}<br>' +
|
| 624 |
'Discussion Resolved Rate: %{y:.2f}%<br>' +
|
| 625 |
'<extra></extra>'
|
|
|
|
| 628 |
)
|
| 629 |
|
| 630 |
# Add bar trace for total discussions (right y-axis)
|
| 631 |
+
# Only show bars for months where assistant has discussions
|
| 632 |
x_bars = []
|
| 633 |
y_bars = []
|
| 634 |
for month, count in zip(months, agent_data['total_discussions']):
|
|
|
|
| 645 |
marker=dict(color=color, opacity=0.6),
|
| 646 |
legendgroup=agent_name,
|
| 647 |
showlegend=False, # Hide duplicate legend entry (already shown in Scatter)
|
| 648 |
+
hovertemplate='<b>Assistant: %{fullData.name}</b><br>' +
|
| 649 |
'Month: %{x}<br>' +
|
| 650 |
'Total Discussions: %{y}<br>' +
|
| 651 |
'<extra></extra>',
|
| 652 |
+
offsetgroup=agent_name # Group bars by assistant for proper spacing
|
| 653 |
),
|
| 654 |
secondary_y=True
|
| 655 |
)
|
|
|
|
| 671 |
show_legend = (top_n is not None and top_n <= 10)
|
| 672 |
fig.update_layout(
|
| 673 |
title=None,
|
| 674 |
+
hovermode='closest', # Show individual assistant info on hover
|
| 675 |
barmode='group',
|
| 676 |
height=600,
|
| 677 |
showlegend=show_legend,
|
|
|
|
| 710 |
filtered_count = 0
|
| 711 |
for identifier, data in cache_dict.items():
|
| 712 |
total_issues = data.get('total_issues', 0)
|
| 713 |
+
print(f" Assistant '{identifier}': {total_issues} issues")
|
| 714 |
|
| 715 |
+
# Filter out assistants with zero total issues
|
| 716 |
if total_issues == 0:
|
| 717 |
filtered_count += 1
|
| 718 |
continue
|
|
|
|
| 730 |
data.get('resolved_discussions', 0), # Resolved Discussions
|
| 731 |
])
|
| 732 |
|
| 733 |
+
print(f"Filtered out {filtered_count} assistants with 0 issues")
|
| 734 |
+
print(f"Leaderboard will show {len(rows)} assistants")
|
| 735 |
|
| 736 |
# Create DataFrame
|
| 737 |
column_names = [col[0] for col in LEADERBOARD_COLUMNS]
|
|
|
|
| 807 |
|
| 808 |
def submit_agent(identifier, agent_name, organization, website):
|
| 809 |
"""
|
| 810 |
+
Submit a new assistant to the leaderboard.
|
| 811 |
Validates input and saves submission.
|
| 812 |
"""
|
| 813 |
# Validate required fields
|
| 814 |
if not identifier or not identifier.strip():
|
| 815 |
return "ERROR: GitHub identifier is required", gr.update()
|
| 816 |
if not agent_name or not agent_name.strip():
|
| 817 |
+
return "ERROR: Assistant name is required", gr.update()
|
| 818 |
if not organization or not organization.strip():
|
| 819 |
return "ERROR: Organization name is required", gr.update()
|
| 820 |
if not website or not website.strip():
|
|
|
|
| 831 |
if not is_valid:
|
| 832 |
return f"ERROR: {message}", gr.update()
|
| 833 |
|
| 834 |
+
# Check for duplicates by loading assistants from HuggingFace
|
| 835 |
+
assistants = load_agents_from_hf()
|
| 836 |
+
if assistants:
|
| 837 |
+
existing_names = {assistant['github_identifier'] for assistant in assistants}
|
| 838 |
if identifier in existing_names:
|
| 839 |
+
return f"WARNING: Assistant with identifier '{identifier}' already exists", gr.update()
|
| 840 |
|
| 841 |
# Create submission
|
| 842 |
submission = {
|
|
|
|
| 873 |
if data:
|
| 874 |
print(f"Successfully reloaded leaderboard data")
|
| 875 |
print(f" Last updated: {data.get('metadata', {}).get('last_updated', 'Unknown')}")
|
| 876 |
+
print(f" Assistants: {len(data.get('leaderboard', {}))}")
|
| 877 |
else:
|
| 878 |
print(f"No data available")
|
| 879 |
except Exception as e:
|
|
|
|
| 886 |
# GRADIO APPLICATION
|
| 887 |
# =============================================================================
|
| 888 |
|
| 889 |
+
print(f"\nStarting SWE Assistant Issue Leaderboard")
|
| 890 |
print(f" Data source: {LEADERBOARD_REPO}")
|
| 891 |
print(f" Reload frequency: Daily at 12:00 AM UTC\n")
|
| 892 |
|
|
|
|
| 907 |
print(f"{'='*80}\n")
|
| 908 |
|
| 909 |
# Create Gradio interface
|
| 910 |
+
with gr.Blocks(title="SWE Assistant Issue & Discussion Leaderboard", theme=gr.themes.Soft()) as app:
|
| 911 |
+
gr.Markdown("# SWE Assistant Issue & Discussion Leaderboard")
|
| 912 |
+
gr.Markdown(f"Track and compare GitHub issue and discussion resolution statistics for SWE assistants")
|
| 913 |
|
| 914 |
with gr.Tabs():
|
| 915 |
|
| 916 |
# Leaderboard Tab
|
| 917 |
with gr.Tab("Leaderboard"):
|
| 918 |
+
gr.Markdown("*Statistics are based on assistant issue resolution activity tracked by the system*")
|
| 919 |
leaderboard_table = Leaderboard(
|
| 920 |
value=pd.DataFrame(columns=[col[0] for col in LEADERBOARD_COLUMNS]), # Empty initially
|
| 921 |
datatype=LEADERBOARD_COLUMNS,
|
| 922 |
+
search_columns=["Assistant", "Website"],
|
| 923 |
filter_columns=[
|
| 924 |
ColumnFilter(
|
| 925 |
"Issue Resolved Rate (%)",
|
|
|
|
| 942 |
# Monthly Metrics Section
|
| 943 |
gr.Markdown("---") # Divider
|
| 944 |
with gr.Group():
|
| 945 |
+
gr.Markdown("### Issue Monthly Performance - Top 5 Assistants")
|
| 946 |
+
gr.Markdown("*Shows issue resolution trends and volumes for the most active assistants*")
|
| 947 |
monthly_metrics_plot = gr.Plot(label="Issue Monthly Metrics")
|
| 948 |
|
| 949 |
# Load monthly metrics when app starts
|
|
|
|
| 956 |
# Discussion Monthly Metrics Section
|
| 957 |
gr.Markdown("---") # Divider
|
| 958 |
with gr.Group():
|
| 959 |
+
gr.Markdown("### Discussion Monthly Performance - Top 5 Assistants")
|
| 960 |
+
gr.Markdown("*Shows discussion resolution trends and volumes for the most active assistants*")
|
| 961 |
discussion_metrics_plot = gr.Plot(label="Discussion Monthly Metrics")
|
| 962 |
|
| 963 |
# Load discussion monthly metrics when app starts
|
|
|
|
| 987 |
)
|
| 988 |
|
| 989 |
|
| 990 |
+
# Submit Assistant Tab
|
| 991 |
+
with gr.Tab("Submit Your Assistant"):
|
| 992 |
|
| 993 |
+
gr.Markdown("Fill in the details below to add your assistant to the leaderboard.")
|
| 994 |
|
| 995 |
with gr.Row():
|
| 996 |
with gr.Column():
|
| 997 |
github_input = gr.Textbox(
|
| 998 |
label="GitHub Identifier*",
|
| 999 |
+
placeholder="Your assistant username (e.g., my-assistant[bot])"
|
| 1000 |
)
|
| 1001 |
name_input = gr.Textbox(
|
| 1002 |
+
label="Assistant Name*",
|
| 1003 |
+
placeholder="Your assistant's display name"
|
| 1004 |
)
|
| 1005 |
|
| 1006 |
with gr.Column():
|
|
|
|
| 1010 |
)
|
| 1011 |
website_input = gr.Textbox(
|
| 1012 |
label="Website*",
|
| 1013 |
+
placeholder="https://your-assistant-website.com"
|
| 1014 |
)
|
| 1015 |
|
| 1016 |
submit_button = gr.Button(
|
| 1017 |
+
"Submit Assistant",
|
| 1018 |
variant="primary"
|
| 1019 |
)
|
| 1020 |
submission_status = gr.Textbox(
|
msr.py
CHANGED
|
@@ -361,14 +361,14 @@ def generate_file_path_patterns(start_date, end_date, data_dir=GHARCHIVE_DATA_LO
|
|
| 361 |
def fetch_all_metadata_streaming(conn, identifiers, start_date, end_date):
|
| 362 |
"""
|
| 363 |
UNIFIED QUERY: Fetches ALL metadata types in ONE query per batch:
|
| 364 |
-
- IssuesEvent, IssueCommentEvent (for
|
| 365 |
- PullRequestEvent (for wanted issue tracking)
|
| 366 |
- DiscussionEvent (for discussion tracking)
|
| 367 |
|
| 368 |
Then post-processes in Python to separate into:
|
| 369 |
-
1.
|
| 370 |
-
2. Wanted issues: Long-standing issues from tracked orgs linked to merged PRs by
|
| 371 |
-
3. Discussions: GitHub discussions created by
|
| 372 |
|
| 373 |
This approach is more efficient than running separate queries for each category.
|
| 374 |
|
|
@@ -380,17 +380,17 @@ def fetch_all_metadata_streaming(conn, identifiers, start_date, end_date):
|
|
| 380 |
|
| 381 |
Returns:
|
| 382 |
Dictionary with four keys:
|
| 383 |
-
- 'agent_issues': {agent_id: [issue_metadata]} for
|
| 384 |
- 'wanted_open': [open_wanted_issues] for long-standing open issues
|
| 385 |
- 'wanted_resolved': {agent_id: [resolved_wanted]} for resolved wanted issues
|
| 386 |
-
- 'agent_discussions': {agent_id: [discussion_metadata]} for
|
| 387 |
"""
|
| 388 |
print(f" Fetching ALL metadata (issues, PRs, discussions) with unified query...")
|
| 389 |
identifier_set = set(identifiers)
|
| 390 |
identifier_list = ', '.join([f"'{id}'" for id in identifiers])
|
| 391 |
tracked_orgs_list = ', '.join([f"'{org}'" for org in TRACKED_ORGS])
|
| 392 |
|
| 393 |
-
# Storage for
|
| 394 |
agent_issues = defaultdict(list) # agent_id -> [issue_metadata]
|
| 395 |
agent_issue_urls = defaultdict(set) # agent_id -> set of issue URLs (for deduplication)
|
| 396 |
|
|
@@ -433,7 +433,7 @@ def fetch_all_metadata_streaming(conn, identifiers, start_date, end_date):
|
|
| 433 |
|
| 434 |
try:
|
| 435 |
# UNIFIED QUERY: Fetch ALL event types in ONE query
|
| 436 |
-
# Post-process in Python to separate into
|
| 437 |
unified_query = f"""
|
| 438 |
SELECT
|
| 439 |
type,
|
|
@@ -448,7 +448,7 @@ def fetch_all_metadata_streaming(conn, identifiers, start_date, end_date):
|
|
| 448 |
json_extract(payload, '$.issue.labels') as issue_labels,
|
| 449 |
json_extract_string(payload, '$.issue.pull_request') as is_pull_request,
|
| 450 |
json_extract_string(payload, '$.issue.state_reason') as issue_state_reason,
|
| 451 |
-
-- Actor/assignee fields for
|
| 452 |
json_extract_string(payload, '$.issue.user.login') as issue_creator,
|
| 453 |
json_extract_string(payload, '$.issue.assignee.login') as issue_assignee,
|
| 454 |
json_extract(payload, '$.issue.assignees') as issue_assignees,
|
|
@@ -481,7 +481,7 @@ def fetch_all_metadata_streaming(conn, identifiers, start_date, end_date):
|
|
| 481 |
WHERE
|
| 482 |
type IN ('IssuesEvent', 'IssueCommentEvent', 'PullRequestEvent', 'DiscussionEvent')
|
| 483 |
AND (
|
| 484 |
-
--
|
| 485 |
(type = 'IssuesEvent' AND (
|
| 486 |
json_extract_string(payload, '$.issue.user.login') IN ({identifier_list})
|
| 487 |
OR json_extract_string(payload, '$.issue.assignee.login') IN ({identifier_list})
|
|
@@ -491,17 +491,17 @@ def fetch_all_metadata_streaming(conn, identifiers, start_date, end_date):
|
|
| 491 |
)
|
| 492 |
OR SPLIT_PART(json_extract_string(repo, '$.name'), '/', 1) IN ({tracked_orgs_list})
|
| 493 |
))
|
| 494 |
-
-- Issue comments:
|
| 495 |
OR (type = 'IssueCommentEvent' AND (
|
| 496 |
json_extract_string(payload, '$.comment.user.login') IN ({identifier_list})
|
| 497 |
OR SPLIT_PART(json_extract_string(repo, '$.name'), '/', 1) IN ({tracked_orgs_list})
|
| 498 |
))
|
| 499 |
-
-- PRs:
|
| 500 |
OR (type = 'PullRequestEvent' AND (
|
| 501 |
json_extract_string(payload, '$.pull_request.user.login') IN ({identifier_list})
|
| 502 |
OR SPLIT_PART(json_extract_string(repo, '$.name'), '/', 1) IN ({tracked_orgs_list})
|
| 503 |
))
|
| 504 |
-
-- Discussions:
|
| 505 |
OR (type = 'DiscussionEvent'
|
| 506 |
AND json_extract_string(payload, '$.discussion.user.login') IN ({identifier_list})
|
| 507 |
AND SPLIT_PART(json_extract_string(repo, '$.name'), '/', 1) IN ({tracked_orgs_list})
|
|
@@ -522,7 +522,7 @@ def fetch_all_metadata_streaming(conn, identifiers, start_date, end_date):
|
|
| 522 |
issue_events = [] # For wanted tracking
|
| 523 |
pr_events = [] # For wanted tracking
|
| 524 |
discussion_events = [] # For discussion tracking
|
| 525 |
-
agent_issue_events = [] # For
|
| 526 |
|
| 527 |
for row in all_results:
|
| 528 |
event_type = row[0]
|
|
@@ -530,7 +530,7 @@ def fetch_all_metadata_streaming(conn, identifiers, start_date, end_date):
|
|
| 530 |
|
| 531 |
if event_type in ('IssuesEvent', 'IssueCommentEvent'):
|
| 532 |
if not is_pr: # It's an issue, not a PR
|
| 533 |
-
# Check if this is an
|
| 534 |
issue_creator = row[11]
|
| 535 |
issue_assignee = row[12]
|
| 536 |
issue_assignees_json = row[13]
|
|
@@ -564,7 +564,7 @@ def fetch_all_metadata_streaming(conn, identifiers, start_date, end_date):
|
|
| 564 |
pass
|
| 565 |
|
| 566 |
elif event_type == 'IssueCommentEvent':
|
| 567 |
-
# Check if commenter is an
|
| 568 |
if commenter in identifier_set:
|
| 569 |
agent_identifier = commenter
|
| 570 |
|
|
@@ -584,7 +584,7 @@ def fetch_all_metadata_streaming(conn, identifiers, start_date, end_date):
|
|
| 584 |
elif event_type == 'DiscussionEvent':
|
| 585 |
discussion_events.append(row)
|
| 586 |
|
| 587 |
-
# Process
|
| 588 |
for row, agent_identifier in agent_issue_events:
|
| 589 |
# Row indices: repo_url=2, issue_url=3, issue_created_at=6, issue_closed_at=7, issue_state_reason=10
|
| 590 |
repo_url = row[2]
|
|
@@ -612,7 +612,7 @@ def fetch_all_metadata_streaming(conn, identifiers, start_date, end_date):
|
|
| 612 |
except:
|
| 613 |
continue
|
| 614 |
|
| 615 |
-
# Deduplicate: only add if we haven't seen this issue for this
|
| 616 |
if full_url in agent_issue_urls[agent_identifier]:
|
| 617 |
continue
|
| 618 |
|
|
@@ -753,8 +753,8 @@ def fetch_all_metadata_streaming(conn, identifiers, start_date, end_date):
|
|
| 753 |
except:
|
| 754 |
continue
|
| 755 |
|
| 756 |
-
# Group by creator (
|
| 757 |
-
# Only track discussions from our
|
| 758 |
if discussion_creator not in identifier_set:
|
| 759 |
continue
|
| 760 |
|
|
@@ -776,12 +776,12 @@ def fetch_all_metadata_streaming(conn, identifiers, start_date, end_date):
|
|
| 776 |
'state_reason': discussion_state_reason
|
| 777 |
}
|
| 778 |
|
| 779 |
-
# Group by
|
| 780 |
if discussion_creator not in discussions_by_agent:
|
| 781 |
discussions_by_agent[discussion_creator] = []
|
| 782 |
discussions_by_agent[discussion_creator].append(discussion_meta)
|
| 783 |
|
| 784 |
-
print(f"✓ {len(agent_issue_events)}
|
| 785 |
|
| 786 |
except Exception as e:
|
| 787 |
print(f"\n ✗ Batch {batch_num} error: {str(e)}")
|
|
@@ -790,7 +790,7 @@ def fetch_all_metadata_streaming(conn, identifiers, start_date, end_date):
|
|
| 790 |
# Move to next batch
|
| 791 |
current_date = batch_end + timedelta(days=1)
|
| 792 |
|
| 793 |
-
# Post-processing: Filter issues and assign to
|
| 794 |
print(f"\n Post-processing {len(all_issues)} wanted issues...")
|
| 795 |
|
| 796 |
wanted_open = []
|
|
@@ -803,7 +803,7 @@ def fetch_all_metadata_streaming(conn, identifiers, start_date, end_date):
|
|
| 803 |
if not linked_prs:
|
| 804 |
continue
|
| 805 |
|
| 806 |
-
# Check if any linked PR was merged AND created by an
|
| 807 |
resolved_by = None
|
| 808 |
for pr_url in linked_prs:
|
| 809 |
merged_at = pr_merged_at.get(pr_url)
|
|
@@ -862,10 +862,10 @@ def fetch_all_metadata_streaming(conn, identifiers, start_date, end_date):
|
|
| 862 |
except:
|
| 863 |
pass
|
| 864 |
|
| 865 |
-
print(f" ✓ Found {sum(len(issues) for issues in agent_issues.values())}
|
| 866 |
print(f" ✓ Found {len(wanted_open)} long-standing open wanted issues")
|
| 867 |
-
print(f" ✓ Found {sum(len(issues) for issues in wanted_resolved.values())} resolved wanted issues across {len(wanted_resolved)}
|
| 868 |
-
print(f" ✓ Found {sum(len(discussions) for discussions in discussions_by_agent.values())} discussions across {len(discussions_by_agent)}
|
| 869 |
|
| 870 |
return {
|
| 871 |
'agent_issues': dict(agent_issues),
|
|
@@ -933,14 +933,14 @@ def sync_agents_repo():
|
|
| 933 |
|
| 934 |
def load_agents_from_hf():
|
| 935 |
"""
|
| 936 |
-
Load all
|
| 937 |
ALWAYS syncs with remote first to ensure we have the latest bot data.
|
| 938 |
"""
|
| 939 |
# MANDATORY: Sync with remote first to get latest bot data
|
| 940 |
-
print(f" Syncing bot_metadata repository to get latest
|
| 941 |
sync_agents_repo() # Will raise exception if sync fails
|
| 942 |
|
| 943 |
-
|
| 944 |
|
| 945 |
# Scan local directory for JSON files
|
| 946 |
if not os.path.exists(AGENTS_REPO_LOCAL_PATH):
|
|
@@ -948,7 +948,7 @@ def load_agents_from_hf():
|
|
| 948 |
|
| 949 |
# Walk through the directory to find all JSON files
|
| 950 |
files_processed = 0
|
| 951 |
-
print(f" Loading
|
| 952 |
|
| 953 |
for root, dirs, files in os.walk(AGENTS_REPO_LOCAL_PATH):
|
| 954 |
# Skip .git directory
|
|
@@ -966,7 +966,7 @@ def load_agents_from_hf():
|
|
| 966 |
with open(file_path, 'r', encoding='utf-8') as f:
|
| 967 |
agent_data = json.load(f)
|
| 968 |
|
| 969 |
-
# Only include active
|
| 970 |
if agent_data.get('status') != 'active':
|
| 971 |
continue
|
| 972 |
|
|
@@ -974,14 +974,14 @@ def load_agents_from_hf():
|
|
| 974 |
github_identifier = filename.replace('.json', '')
|
| 975 |
agent_data['github_identifier'] = github_identifier
|
| 976 |
|
| 977 |
-
|
| 978 |
|
| 979 |
except Exception as e:
|
| 980 |
print(f" ⚠ Error loading {filename}: {str(e)}")
|
| 981 |
continue
|
| 982 |
|
| 983 |
-
print(f" ✓ Loaded {len(
|
| 984 |
-
return
|
| 985 |
|
| 986 |
|
| 987 |
def calculate_issue_stats_from_metadata(metadata_list):
|
|
@@ -1002,12 +1002,12 @@ def calculate_issue_stats_from_metadata(metadata_list):
|
|
| 1002 |
}
|
| 1003 |
|
| 1004 |
|
| 1005 |
-
def calculate_monthly_metrics_by_agent(all_metadata_dict,
|
| 1006 |
-
"""Calculate monthly metrics for all
|
| 1007 |
-
identifier_to_name = {
|
| 1008 |
|
| 1009 |
if not all_metadata_dict:
|
| 1010 |
-
return {'
|
| 1011 |
|
| 1012 |
agent_month_data = defaultdict(lambda: defaultdict(list))
|
| 1013 |
|
|
@@ -1065,7 +1065,7 @@ def calculate_monthly_metrics_by_agent(all_metadata_dict, agents):
|
|
| 1065 |
agents_list = sorted(list(agent_month_data.keys()))
|
| 1066 |
|
| 1067 |
return {
|
| 1068 |
-
'
|
| 1069 |
'months': months,
|
| 1070 |
'data': result_data
|
| 1071 |
}
|
|
@@ -1086,12 +1086,12 @@ def calculate_discussion_stats_from_metadata(metadata_list):
|
|
| 1086 |
}
|
| 1087 |
|
| 1088 |
|
| 1089 |
-
def calculate_monthly_metrics_by_agent_discussions(all_discussions_dict,
|
| 1090 |
-
"""Calculate monthly metrics for discussions for all
|
| 1091 |
-
identifier_to_name = {
|
| 1092 |
|
| 1093 |
if not all_discussions_dict:
|
| 1094 |
-
return {'
|
| 1095 |
|
| 1096 |
agent_month_data = defaultdict(lambda: defaultdict(list))
|
| 1097 |
|
|
@@ -1145,23 +1145,23 @@ def calculate_monthly_metrics_by_agent_discussions(all_discussions_dict, agents)
|
|
| 1145 |
agents_list = sorted(list(agent_month_data.keys()))
|
| 1146 |
|
| 1147 |
return {
|
| 1148 |
-
'
|
| 1149 |
'months': months,
|
| 1150 |
'data': result_data
|
| 1151 |
}
|
| 1152 |
|
| 1153 |
|
| 1154 |
-
def construct_leaderboard_from_metadata(all_metadata_dict,
|
| 1155 |
"""Construct leaderboard from in-memory issue metadata and discussion metadata.
|
| 1156 |
|
| 1157 |
Args:
|
| 1158 |
-
all_metadata_dict: Dictionary mapping
|
| 1159 |
-
|
| 1160 |
-
wanted_resolved_dict: Optional dictionary mapping
|
| 1161 |
-
discussions_dict: Optional dictionary mapping
|
| 1162 |
"""
|
| 1163 |
-
if not
|
| 1164 |
-
print("Error: No
|
| 1165 |
return {}
|
| 1166 |
|
| 1167 |
if wanted_resolved_dict is None:
|
|
@@ -1172,9 +1172,9 @@ def construct_leaderboard_from_metadata(all_metadata_dict, agents, wanted_resolv
|
|
| 1172 |
|
| 1173 |
cache_dict = {}
|
| 1174 |
|
| 1175 |
-
for
|
| 1176 |
-
identifier =
|
| 1177 |
-
agent_name =
|
| 1178 |
|
| 1179 |
bot_metadata = all_metadata_dict.get(identifier, [])
|
| 1180 |
stats = calculate_issue_stats_from_metadata(bot_metadata)
|
|
@@ -1188,7 +1188,7 @@ def construct_leaderboard_from_metadata(all_metadata_dict, agents, wanted_resolv
|
|
| 1188 |
|
| 1189 |
cache_dict[identifier] = {
|
| 1190 |
'name': agent_name,
|
| 1191 |
-
'website':
|
| 1192 |
'github_identifier': identifier,
|
| 1193 |
**stats,
|
| 1194 |
'resolved_wanted_issues': resolved_wanted,
|
|
@@ -1211,7 +1211,7 @@ def save_leaderboard_data_to_hf(leaderboard_dict, monthly_metrics, wanted_issues
|
|
| 1211 |
wanted_issues = []
|
| 1212 |
|
| 1213 |
if discussion_monthly_metrics is None:
|
| 1214 |
-
discussion_monthly_metrics = {'
|
| 1215 |
|
| 1216 |
combined_data = {
|
| 1217 |
'metadata': {
|
|
@@ -1255,7 +1255,7 @@ def save_leaderboard_data_to_hf(leaderboard_dict, monthly_metrics, wanted_issues
|
|
| 1255 |
|
| 1256 |
def mine_all_agents():
|
| 1257 |
"""
|
| 1258 |
-
Mine issue metadata for all
|
| 1259 |
Downloads GHArchive data, then uses BATCH-based DuckDB queries.
|
| 1260 |
"""
|
| 1261 |
print(f"\n[1/4] Downloading GHArchive data...")
|
|
@@ -1263,19 +1263,19 @@ def mine_all_agents():
|
|
| 1263 |
if not download_all_gharchive_data():
|
| 1264 |
print("Warning: Download had errors, continuing with available data...")
|
| 1265 |
|
| 1266 |
-
print(f"\n[2/4] Loading
|
| 1267 |
|
| 1268 |
-
|
| 1269 |
-
if not
|
| 1270 |
-
print("Error: No
|
| 1271 |
return
|
| 1272 |
|
| 1273 |
-
identifiers = [
|
| 1274 |
if not identifiers:
|
| 1275 |
-
print("Error: No valid
|
| 1276 |
return
|
| 1277 |
|
| 1278 |
-
print(f"\n[3/4] Mining issue metadata ({len(identifiers)}
|
| 1279 |
|
| 1280 |
try:
|
| 1281 |
conn = get_duckdb_connection()
|
|
@@ -1309,11 +1309,11 @@ def mine_all_agents():
|
|
| 1309 |
|
| 1310 |
try:
|
| 1311 |
leaderboard_dict = construct_leaderboard_from_metadata(
|
| 1312 |
-
agent_issues,
|
| 1313 |
)
|
| 1314 |
-
monthly_metrics = calculate_monthly_metrics_by_agent(agent_issues,
|
| 1315 |
discussion_monthly_metrics = calculate_monthly_metrics_by_agent_discussions(
|
| 1316 |
-
agent_discussions,
|
| 1317 |
)
|
| 1318 |
save_leaderboard_data_to_hf(
|
| 1319 |
leaderboard_dict, monthly_metrics, wanted_open, discussion_monthly_metrics
|
|
@@ -1350,7 +1350,7 @@ def setup_scheduler():
|
|
| 1350 |
mine_all_agents,
|
| 1351 |
trigger=trigger,
|
| 1352 |
id='mine_all_agents',
|
| 1353 |
-
name='Mine GHArchive data for all
|
| 1354 |
replace_existing=True
|
| 1355 |
)
|
| 1356 |
|
|
|
|
| 361 |
def fetch_all_metadata_streaming(conn, identifiers, start_date, end_date):
|
| 362 |
"""
|
| 363 |
UNIFIED QUERY: Fetches ALL metadata types in ONE query per batch:
|
| 364 |
+
- IssuesEvent, IssueCommentEvent (for assistant-assigned issues AND wanted issues)
|
| 365 |
- PullRequestEvent (for wanted issue tracking)
|
| 366 |
- DiscussionEvent (for discussion tracking)
|
| 367 |
|
| 368 |
Then post-processes in Python to separate into:
|
| 369 |
+
1. Assistant-assigned issues: Issues where assistants are assigned to or commented on
|
| 370 |
+
2. Wanted issues: Long-standing issues from tracked orgs linked to merged PRs by assistants
|
| 371 |
+
3. Discussions: GitHub discussions created by assistants
|
| 372 |
|
| 373 |
This approach is more efficient than running separate queries for each category.
|
| 374 |
|
|
|
|
| 380 |
|
| 381 |
Returns:
|
| 382 |
Dictionary with four keys:
|
| 383 |
+
- 'agent_issues': {agent_id: [issue_metadata]} for assistant-assigned issues
|
| 384 |
- 'wanted_open': [open_wanted_issues] for long-standing open issues
|
| 385 |
- 'wanted_resolved': {agent_id: [resolved_wanted]} for resolved wanted issues
|
| 386 |
+
- 'agent_discussions': {agent_id: [discussion_metadata]} for assistant discussions
|
| 387 |
"""
|
| 388 |
print(f" Fetching ALL metadata (issues, PRs, discussions) with unified query...")
|
| 389 |
identifier_set = set(identifiers)
|
| 390 |
identifier_list = ', '.join([f"'{id}'" for id in identifiers])
|
| 391 |
tracked_orgs_list = ', '.join([f"'{org}'" for org in TRACKED_ORGS])
|
| 392 |
|
| 393 |
+
# Storage for assistant-assigned issues
|
| 394 |
agent_issues = defaultdict(list) # agent_id -> [issue_metadata]
|
| 395 |
agent_issue_urls = defaultdict(set) # agent_id -> set of issue URLs (for deduplication)
|
| 396 |
|
|
|
|
| 433 |
|
| 434 |
try:
|
| 435 |
# UNIFIED QUERY: Fetch ALL event types in ONE query
|
| 436 |
+
# Post-process in Python to separate into assistant-assigned issues, wanted issues, PRs, and discussions
|
| 437 |
unified_query = f"""
|
| 438 |
SELECT
|
| 439 |
type,
|
|
|
|
| 448 |
json_extract(payload, '$.issue.labels') as issue_labels,
|
| 449 |
json_extract_string(payload, '$.issue.pull_request') as is_pull_request,
|
| 450 |
json_extract_string(payload, '$.issue.state_reason') as issue_state_reason,
|
| 451 |
+
-- Actor/assignee fields for assistant assignment
|
| 452 |
json_extract_string(payload, '$.issue.user.login') as issue_creator,
|
| 453 |
json_extract_string(payload, '$.issue.assignee.login') as issue_assignee,
|
| 454 |
json_extract(payload, '$.issue.assignees') as issue_assignees,
|
|
|
|
| 481 |
WHERE
|
| 482 |
type IN ('IssuesEvent', 'IssueCommentEvent', 'PullRequestEvent', 'DiscussionEvent')
|
| 483 |
AND (
|
| 484 |
+
-- Assistant-assigned issues: assistant is creator, assignee, or commenter
|
| 485 |
(type = 'IssuesEvent' AND (
|
| 486 |
json_extract_string(payload, '$.issue.user.login') IN ({identifier_list})
|
| 487 |
OR json_extract_string(payload, '$.issue.assignee.login') IN ({identifier_list})
|
|
|
|
| 491 |
)
|
| 492 |
OR SPLIT_PART(json_extract_string(repo, '$.name'), '/', 1) IN ({tracked_orgs_list})
|
| 493 |
))
|
| 494 |
+
-- Issue comments: assistant is commenter OR tracked org
|
| 495 |
OR (type = 'IssueCommentEvent' AND (
|
| 496 |
json_extract_string(payload, '$.comment.user.login') IN ({identifier_list})
|
| 497 |
OR SPLIT_PART(json_extract_string(repo, '$.name'), '/', 1) IN ({tracked_orgs_list})
|
| 498 |
))
|
| 499 |
+
-- PRs: assistant is creator OR tracked org (for wanted issue tracking)
|
| 500 |
OR (type = 'PullRequestEvent' AND (
|
| 501 |
json_extract_string(payload, '$.pull_request.user.login') IN ({identifier_list})
|
| 502 |
OR SPLIT_PART(json_extract_string(repo, '$.name'), '/', 1) IN ({tracked_orgs_list})
|
| 503 |
))
|
| 504 |
+
-- Discussions: assistant is creator AND tracked org
|
| 505 |
OR (type = 'DiscussionEvent'
|
| 506 |
AND json_extract_string(payload, '$.discussion.user.login') IN ({identifier_list})
|
| 507 |
AND SPLIT_PART(json_extract_string(repo, '$.name'), '/', 1) IN ({tracked_orgs_list})
|
|
|
|
| 522 |
issue_events = [] # For wanted tracking
|
| 523 |
pr_events = [] # For wanted tracking
|
| 524 |
discussion_events = [] # For discussion tracking
|
| 525 |
+
agent_issue_events = [] # For assistant-assigned issues
|
| 526 |
|
| 527 |
for row in all_results:
|
| 528 |
event_type = row[0]
|
|
|
|
| 530 |
|
| 531 |
if event_type in ('IssuesEvent', 'IssueCommentEvent'):
|
| 532 |
if not is_pr: # It's an issue, not a PR
|
| 533 |
+
# Check if this is an assistant-assigned issue
|
| 534 |
issue_creator = row[11]
|
| 535 |
issue_assignee = row[12]
|
| 536 |
issue_assignees_json = row[13]
|
|
|
|
| 564 |
pass
|
| 565 |
|
| 566 |
elif event_type == 'IssueCommentEvent':
|
| 567 |
+
# Check if commenter is an assistant
|
| 568 |
if commenter in identifier_set:
|
| 569 |
agent_identifier = commenter
|
| 570 |
|
|
|
|
| 584 |
elif event_type == 'DiscussionEvent':
|
| 585 |
discussion_events.append(row)
|
| 586 |
|
| 587 |
+
# Process assistant-assigned issues
|
| 588 |
for row, agent_identifier in agent_issue_events:
|
| 589 |
# Row indices: repo_url=2, issue_url=3, issue_created_at=6, issue_closed_at=7, issue_state_reason=10
|
| 590 |
repo_url = row[2]
|
|
|
|
| 612 |
except:
|
| 613 |
continue
|
| 614 |
|
| 615 |
+
# Deduplicate: only add if we haven't seen this issue for this assistant
|
| 616 |
if full_url in agent_issue_urls[agent_identifier]:
|
| 617 |
continue
|
| 618 |
|
|
|
|
| 753 |
except:
|
| 754 |
continue
|
| 755 |
|
| 756 |
+
# Group by creator (assistant identifier)
|
| 757 |
+
# Only track discussions from our assistant identifiers
|
| 758 |
if discussion_creator not in identifier_set:
|
| 759 |
continue
|
| 760 |
|
|
|
|
| 776 |
'state_reason': discussion_state_reason
|
| 777 |
}
|
| 778 |
|
| 779 |
+
# Group by assistant
|
| 780 |
if discussion_creator not in discussions_by_agent:
|
| 781 |
discussions_by_agent[discussion_creator] = []
|
| 782 |
discussions_by_agent[discussion_creator].append(discussion_meta)
|
| 783 |
|
| 784 |
+
print(f"✓ {len(agent_issue_events)} assistant issues, {len(issue_events)} wanted issues, {len(pr_events)} PRs, {len(discussion_events)} discussions")
|
| 785 |
|
| 786 |
except Exception as e:
|
| 787 |
print(f"\n ✗ Batch {batch_num} error: {str(e)}")
|
|
|
|
| 790 |
# Move to next batch
|
| 791 |
current_date = batch_end + timedelta(days=1)
|
| 792 |
|
| 793 |
+
# Post-processing: Filter issues and assign to assistants
|
| 794 |
print(f"\n Post-processing {len(all_issues)} wanted issues...")
|
| 795 |
|
| 796 |
wanted_open = []
|
|
|
|
| 803 |
if not linked_prs:
|
| 804 |
continue
|
| 805 |
|
| 806 |
+
# Check if any linked PR was merged AND created by an assistant
|
| 807 |
resolved_by = None
|
| 808 |
for pr_url in linked_prs:
|
| 809 |
merged_at = pr_merged_at.get(pr_url)
|
|
|
|
| 862 |
except:
|
| 863 |
pass
|
| 864 |
|
| 865 |
+
print(f" ✓ Found {sum(len(issues) for issues in agent_issues.values())} assistant-assigned issues across {len(agent_issues)} assistants")
|
| 866 |
print(f" ✓ Found {len(wanted_open)} long-standing open wanted issues")
|
| 867 |
+
print(f" ✓ Found {sum(len(issues) for issues in wanted_resolved.values())} resolved wanted issues across {len(wanted_resolved)} assistants")
|
| 868 |
+
print(f" ✓ Found {sum(len(discussions) for discussions in discussions_by_agent.values())} discussions across {len(discussions_by_agent)} assistants")
|
| 869 |
|
| 870 |
return {
|
| 871 |
'agent_issues': dict(agent_issues),
|
|
|
|
| 933 |
|
| 934 |
def load_agents_from_hf():
|
| 935 |
"""
|
| 936 |
+
Load all assistant metadata JSON files from local git repository.
|
| 937 |
ALWAYS syncs with remote first to ensure we have the latest bot data.
|
| 938 |
"""
|
| 939 |
# MANDATORY: Sync with remote first to get latest bot data
|
| 940 |
+
print(f" Syncing bot_metadata repository to get latest assistants...")
|
| 941 |
sync_agents_repo() # Will raise exception if sync fails
|
| 942 |
|
| 943 |
+
assistants = []
|
| 944 |
|
| 945 |
# Scan local directory for JSON files
|
| 946 |
if not os.path.exists(AGENTS_REPO_LOCAL_PATH):
|
|
|
|
| 948 |
|
| 949 |
# Walk through the directory to find all JSON files
|
| 950 |
files_processed = 0
|
| 951 |
+
print(f" Loading assistant metadata from {AGENTS_REPO_LOCAL_PATH}...")
|
| 952 |
|
| 953 |
for root, dirs, files in os.walk(AGENTS_REPO_LOCAL_PATH):
|
| 954 |
# Skip .git directory
|
|
|
|
| 966 |
with open(file_path, 'r', encoding='utf-8') as f:
|
| 967 |
agent_data = json.load(f)
|
| 968 |
|
| 969 |
+
# Only include active assistants
|
| 970 |
if agent_data.get('status') != 'active':
|
| 971 |
continue
|
| 972 |
|
|
|
|
| 974 |
github_identifier = filename.replace('.json', '')
|
| 975 |
agent_data['github_identifier'] = github_identifier
|
| 976 |
|
| 977 |
+
assistants.append(agent_data)
|
| 978 |
|
| 979 |
except Exception as e:
|
| 980 |
print(f" ⚠ Error loading {filename}: {str(e)}")
|
| 981 |
continue
|
| 982 |
|
| 983 |
+
print(f" ✓ Loaded {len(assistants)} active assistants (from {files_processed} total files)")
|
| 984 |
+
return assistants
|
| 985 |
|
| 986 |
|
| 987 |
def calculate_issue_stats_from_metadata(metadata_list):
|
|
|
|
| 1002 |
}
|
| 1003 |
|
| 1004 |
|
| 1005 |
+
def calculate_monthly_metrics_by_agent(all_metadata_dict, assistants):
|
| 1006 |
+
"""Calculate monthly metrics for all assistants for visualization."""
|
| 1007 |
+
identifier_to_name = {assistant.get('github_identifier'): assistant.get('name') for assistant in assistants if assistant.get('github_identifier')}
|
| 1008 |
|
| 1009 |
if not all_metadata_dict:
|
| 1010 |
+
return {'assistants': [], 'months': [], 'data': {}}
|
| 1011 |
|
| 1012 |
agent_month_data = defaultdict(lambda: defaultdict(list))
|
| 1013 |
|
|
|
|
| 1065 |
agents_list = sorted(list(agent_month_data.keys()))
|
| 1066 |
|
| 1067 |
return {
|
| 1068 |
+
'assistants': agents_list,
|
| 1069 |
'months': months,
|
| 1070 |
'data': result_data
|
| 1071 |
}
|
|
|
|
| 1086 |
}
|
| 1087 |
|
| 1088 |
|
| 1089 |
+
def calculate_monthly_metrics_by_agent_discussions(all_discussions_dict, assistants):
|
| 1090 |
+
"""Calculate monthly metrics for discussions for all assistants for visualization."""
|
| 1091 |
+
identifier_to_name = {assistant.get('github_identifier'): assistant.get('name') for assistant in assistants if assistant.get('github_identifier')}
|
| 1092 |
|
| 1093 |
if not all_discussions_dict:
|
| 1094 |
+
return {'assistants': [], 'months': [], 'data': {}}
|
| 1095 |
|
| 1096 |
agent_month_data = defaultdict(lambda: defaultdict(list))
|
| 1097 |
|
|
|
|
| 1145 |
agents_list = sorted(list(agent_month_data.keys()))
|
| 1146 |
|
| 1147 |
return {
|
| 1148 |
+
'assistants': agents_list,
|
| 1149 |
'months': months,
|
| 1150 |
'data': result_data
|
| 1151 |
}
|
| 1152 |
|
| 1153 |
|
| 1154 |
+
def construct_leaderboard_from_metadata(all_metadata_dict, assistants, wanted_resolved_dict=None, discussions_dict=None):
|
| 1155 |
"""Construct leaderboard from in-memory issue metadata and discussion metadata.
|
| 1156 |
|
| 1157 |
Args:
|
| 1158 |
+
all_metadata_dict: Dictionary mapping assistant ID to list of issue metadata (assistant-assigned issues)
|
| 1159 |
+
assistants: List of assistant metadata
|
| 1160 |
+
wanted_resolved_dict: Optional dictionary mapping assistant ID to list of resolved wanted issues
|
| 1161 |
+
discussions_dict: Optional dictionary mapping assistant ID to list of discussion metadata
|
| 1162 |
"""
|
| 1163 |
+
if not assistants:
|
| 1164 |
+
print("Error: No assistants found")
|
| 1165 |
return {}
|
| 1166 |
|
| 1167 |
if wanted_resolved_dict is None:
|
|
|
|
| 1172 |
|
| 1173 |
cache_dict = {}
|
| 1174 |
|
| 1175 |
+
for assistant in assistants:
|
| 1176 |
+
identifier = assistant.get('github_identifier')
|
| 1177 |
+
agent_name = assistant.get('name', 'Unknown')
|
| 1178 |
|
| 1179 |
bot_metadata = all_metadata_dict.get(identifier, [])
|
| 1180 |
stats = calculate_issue_stats_from_metadata(bot_metadata)
|
|
|
|
| 1188 |
|
| 1189 |
cache_dict[identifier] = {
|
| 1190 |
'name': agent_name,
|
| 1191 |
+
'website': assistant.get('website', 'N/A'),
|
| 1192 |
'github_identifier': identifier,
|
| 1193 |
**stats,
|
| 1194 |
'resolved_wanted_issues': resolved_wanted,
|
|
|
|
| 1211 |
wanted_issues = []
|
| 1212 |
|
| 1213 |
if discussion_monthly_metrics is None:
|
| 1214 |
+
discussion_monthly_metrics = {'assistants': [], 'months': [], 'data': {}}
|
| 1215 |
|
| 1216 |
combined_data = {
|
| 1217 |
'metadata': {
|
|
|
|
| 1255 |
|
| 1256 |
def mine_all_agents():
|
| 1257 |
"""
|
| 1258 |
+
Mine issue metadata for all assistants using STREAMING batch processing.
|
| 1259 |
Downloads GHArchive data, then uses BATCH-based DuckDB queries.
|
| 1260 |
"""
|
| 1261 |
print(f"\n[1/4] Downloading GHArchive data...")
|
|
|
|
| 1263 |
if not download_all_gharchive_data():
|
| 1264 |
print("Warning: Download had errors, continuing with available data...")
|
| 1265 |
|
| 1266 |
+
print(f"\n[2/4] Loading assistant metadata...")
|
| 1267 |
|
| 1268 |
+
assistants = load_agents_from_hf()
|
| 1269 |
+
if not assistants:
|
| 1270 |
+
print("Error: No assistants found")
|
| 1271 |
return
|
| 1272 |
|
| 1273 |
+
identifiers = [assistant['github_identifier'] for assistant in assistants if assistant.get('github_identifier')]
|
| 1274 |
if not identifiers:
|
| 1275 |
+
print("Error: No valid assistant identifiers found")
|
| 1276 |
return
|
| 1277 |
|
| 1278 |
+
print(f"\n[3/4] Mining issue metadata ({len(identifiers)} assistants, {LEADERBOARD_TIME_FRAME_DAYS} days)...")
|
| 1279 |
|
| 1280 |
try:
|
| 1281 |
conn = get_duckdb_connection()
|
|
|
|
| 1309 |
|
| 1310 |
try:
|
| 1311 |
leaderboard_dict = construct_leaderboard_from_metadata(
|
| 1312 |
+
agent_issues, assistants, wanted_resolved, agent_discussions
|
| 1313 |
)
|
| 1314 |
+
monthly_metrics = calculate_monthly_metrics_by_agent(agent_issues, assistants)
|
| 1315 |
discussion_monthly_metrics = calculate_monthly_metrics_by_agent_discussions(
|
| 1316 |
+
agent_discussions, assistants
|
| 1317 |
)
|
| 1318 |
save_leaderboard_data_to_hf(
|
| 1319 |
leaderboard_dict, monthly_metrics, wanted_open, discussion_monthly_metrics
|
|
|
|
| 1350 |
mine_all_agents,
|
| 1351 |
trigger=trigger,
|
| 1352 |
id='mine_all_agents',
|
| 1353 |
+
name='Mine GHArchive data for all assistants',
|
| 1354 |
replace_existing=True
|
| 1355 |
)
|
| 1356 |
|