zhimin-z commited on
Commit
f51e3c7
·
1 Parent(s): 1bae49b
Files changed (3) hide show
  1. README.md +4 -2
  2. app.py +85 -85
  3. msr.py +69 -69
README.md CHANGED
@@ -28,6 +28,8 @@ If an assistant can consistently resolve issues and discussions across different
28
  Key metrics from the last 180 days:
29
 
30
  **Leaderboard Table**
 
 
31
  - **Issue Resolved Rate (%)**: Percentage of closed issues successfully resolved
32
  - **Discussion Resolved Rate (%)**: Percentage of discussions successfully resolved (answered or closed)
33
  - **Total Issues**: Issues the assistant has been involved with (authored, assigned, or commented on)
@@ -51,7 +53,7 @@ We focus on 180 days to highlight current capabilities and active assistants.
51
  **Data Collection**
52
  We mine GitHub activity from [GHArchive](https://www.gharchive.org/), tracking three types of activities:
53
 
54
- 1. **Agent-Assigned Issues**:
55
  - Issues opened or assigned to the assistant (`IssuesEvent`)
56
  - Issue comments by the assistant (`IssueCommentEvent`)
57
 
@@ -78,7 +80,7 @@ Anyone can submit an assistant. We store metadata in `SWE-Arena/bot_metadata` an
78
  - Searchable table (by assistant name or website)
79
  - Filterable columns (by issue resolved rate, discussion resolved rate)
80
  - Monthly charts (issue and discussion resolution trends and activity)
81
- - View agent-assigned metrics, wanted issue resolutions, and discussion metrics
82
 
83
  **Issues Wanted Tab**:
84
  - Browse long-standing open issues (30+ days) from major open-source projects
 
28
  Key metrics from the last 180 days:
29
 
30
  **Leaderboard Table**
31
+ - **Assistant**: Display name of the assistant
32
+ - **Website**: Link to the assistant's homepage or documentation
33
  - **Issue Resolved Rate (%)**: Percentage of closed issues successfully resolved
34
  - **Discussion Resolved Rate (%)**: Percentage of discussions successfully resolved (answered or closed)
35
  - **Total Issues**: Issues the assistant has been involved with (authored, assigned, or commented on)
 
53
  **Data Collection**
54
  We mine GitHub activity from [GHArchive](https://www.gharchive.org/), tracking three types of activities:
55
 
56
+ 1. **Assistant-Assigned Issues**:
57
  - Issues opened or assigned to the assistant (`IssuesEvent`)
58
  - Issue comments by the assistant (`IssueCommentEvent`)
59
 
 
80
  - Searchable table (by assistant name or website)
81
  - Filterable columns (by issue resolved rate, discussion resolved rate)
82
  - Monthly charts (issue and discussion resolution trends and activity)
83
+ - View assistant-assigned metrics, wanted issue resolutions, and discussion metrics
84
 
85
  **Issues Wanted Tab**:
86
  - Browse long-standing open issues (30+ days) from major open-source projects
app.py CHANGED
@@ -24,7 +24,7 @@ load_dotenv()
24
  # CONFIGURATION
25
  # =============================================================================
26
 
27
- AGENTS_REPO = "SWE-Arena/bot_metadata" # HuggingFace dataset for agent metadata
28
  AGENTS_REPO_LOCAL_PATH = os.path.expanduser("~/bot_metadata") # Local git clone path
29
  LEADERBOARD_FILENAME = f"{os.getenv('COMPOSE_PROJECT_NAME')}.json"
30
  LEADERBOARD_REPO = "SWE-Arena/leaderboard_data" # HuggingFace dataset for leaderboard data
@@ -33,7 +33,7 @@ GIT_SYNC_TIMEOUT = 300 # 5 minutes timeout for git pull
33
  MAX_RETRIES = 5
34
 
35
  LEADERBOARD_COLUMNS = [
36
- ("Agent Name", "string"),
37
  ("Website", "string"),
38
  ("Total Issues", "number"),
39
  ("Total Discussions", "number"),
@@ -162,14 +162,14 @@ def sync_agents_repo():
162
 
163
  def load_agents_from_hf():
164
  """
165
- Load all agent metadata JSON files from local git repository.
166
  ALWAYS syncs with remote first to ensure we have the latest bot data.
167
  """
168
  # MANDATORY: Sync with remote first to get latest bot data
169
- print(f" Syncing bot_metadata repository to get latest agents...")
170
  sync_agents_repo() # Will raise exception if sync fails
171
 
172
- agents = []
173
 
174
  # Scan local directory for JSON files
175
  if not os.path.exists(AGENTS_REPO_LOCAL_PATH):
@@ -177,7 +177,7 @@ def load_agents_from_hf():
177
 
178
  # Walk through the directory to find all JSON files
179
  files_processed = 0
180
- print(f" Loading agent metadata from {AGENTS_REPO_LOCAL_PATH}...")
181
 
182
  for root, dirs, files in os.walk(AGENTS_REPO_LOCAL_PATH):
183
  # Skip .git directory
@@ -195,7 +195,7 @@ def load_agents_from_hf():
195
  with open(file_path, 'r', encoding='utf-8') as f:
196
  agent_data = json.load(f)
197
 
198
- # Only include active agents
199
  if agent_data.get('status') != 'active':
200
  continue
201
 
@@ -203,14 +203,14 @@ def load_agents_from_hf():
203
  github_identifier = filename.replace('.json', '')
204
  agent_data['github_identifier'] = github_identifier
205
 
206
- agents.append(agent_data)
207
 
208
  except Exception as e:
209
  print(f" Warning Error loading {filename}: {str(e)}")
210
  continue
211
 
212
- print(f" Success Loaded {len(agents)} active agents (from {files_processed} total files)")
213
- return agents
214
 
215
 
216
  def get_hf_token():
@@ -265,7 +265,7 @@ def upload_with_retry(api, path_or_fileobj, path_in_repo, repo_id, repo_type, to
265
 
266
 
267
  def save_agent_to_hf(data):
268
- """Save a new agent to HuggingFace dataset as {identifier}.json in root."""
269
  try:
270
  api = HfApi()
271
  token = get_hf_token()
@@ -290,7 +290,7 @@ def save_agent_to_hf(data):
290
  repo_type="dataset",
291
  token=token
292
  )
293
- print(f"Saved agent to HuggingFace: {filename}")
294
  return True
295
  finally:
296
  # Always clean up local file, even if upload fails
@@ -298,7 +298,7 @@ def save_agent_to_hf(data):
298
  os.remove(filename)
299
 
300
  except Exception as e:
301
- print(f"Error saving agent: {str(e)}")
302
  return False
303
 
304
 
@@ -345,10 +345,10 @@ def create_monthly_metrics_plot(top_n=5):
345
  - Left y-axis: Resolved Rate (%) as line curves
346
  - Right y-axis: Total Issues created as bar charts
347
 
348
- Each agent gets a unique color for both their line and bars.
349
 
350
  Args:
351
- top_n: Number of top agents to show (default: 5)
352
  """
353
  # Load from saved dataset
354
  saved_data = load_leaderboard_data_from_hf()
@@ -373,10 +373,10 @@ def create_monthly_metrics_plot(top_n=5):
373
  print(f"Loaded monthly metrics from saved dataset")
374
 
375
  # Apply top_n filter if specified
376
- if top_n is not None and top_n > 0 and metrics.get('agents'):
377
- # Calculate total issues for each agent
378
  agent_totals = []
379
- for agent_name in metrics['agents']:
380
  agent_data = metrics['data'].get(agent_name, {})
381
  total_issues = sum(agent_data.get('total_issues', []))
382
  agent_totals.append((agent_name, total_issues))
@@ -385,14 +385,14 @@ def create_monthly_metrics_plot(top_n=5):
385
  agent_totals.sort(key=lambda x: x[1], reverse=True)
386
  top_agents = [agent_name for agent_name, _ in agent_totals[:top_n]]
387
 
388
- # Filter metrics to only include top agents
389
  metrics = {
390
- 'agents': top_agents,
391
  'months': metrics['months'],
392
- 'data': {agent: metrics['data'][agent] for agent in top_agents if agent in metrics['data']}
393
  }
394
 
395
- if not metrics['agents'] or not metrics['months']:
396
  # Return an empty figure with a message
397
  fig = go.Figure()
398
  fig.add_annotation(
@@ -411,7 +411,7 @@ def create_monthly_metrics_plot(top_n=5):
411
  # Create figure with secondary y-axis
412
  fig = make_subplots(specs=[[{"secondary_y": True}]])
413
 
414
- # Generate unique colors for many agents using HSL color space
415
  def generate_color(index, total):
416
  """Generate distinct colors using HSL color space for better distribution"""
417
  hue = (index * 360 / total) % 360
@@ -419,15 +419,15 @@ def create_monthly_metrics_plot(top_n=5):
419
  lightness = 45 + (index % 2) * 10 # Vary lightness slightly
420
  return f'hsl({hue}, {saturation}%, {lightness}%)'
421
 
422
- agents = metrics['agents']
423
  months = metrics['months']
424
  data = metrics['data']
425
 
426
- # Generate colors for all agents
427
- agent_colors = {agent: generate_color(idx, len(agents)) for idx, agent in enumerate(agents)}
428
 
429
- # Add traces for each agent
430
- for idx, agent_name in enumerate(agents):
431
  color = agent_colors[agent_name]
432
  agent_data = data[agent_name]
433
 
@@ -447,8 +447,8 @@ def create_monthly_metrics_plot(top_n=5):
447
  line=dict(color=color, width=2),
448
  marker=dict(size=8),
449
  legendgroup=agent_name,
450
- showlegend=(top_n is not None and top_n <= 10), # Show legend for top N agents
451
- hovertemplate='<b>Agent: %{fullData.name}</b><br>' +
452
  'Month: %{x}<br>' +
453
  'Resolved Rate: %{y:.2f}%<br>' +
454
  '<extra></extra>'
@@ -457,7 +457,7 @@ def create_monthly_metrics_plot(top_n=5):
457
  )
458
 
459
  # Add bar trace for total issues (right y-axis)
460
- # Only show bars for months where agent has issues
461
  x_bars = []
462
  y_bars = []
463
  for month, count in zip(months, agent_data['total_issues']):
@@ -474,11 +474,11 @@ def create_monthly_metrics_plot(top_n=5):
474
  marker=dict(color=color, opacity=0.6),
475
  legendgroup=agent_name,
476
  showlegend=False, # Hide duplicate legend entry (already shown in Scatter)
477
- hovertemplate='<b>Agent: %{fullData.name}</b><br>' +
478
  'Month: %{x}<br>' +
479
  'Total Issues: %{y}<br>' +
480
  '<extra></extra>',
481
- offsetgroup=agent_name # Group bars by agent for proper spacing
482
  ),
483
  secondary_y=True
484
  )
@@ -500,7 +500,7 @@ def create_monthly_metrics_plot(top_n=5):
500
  show_legend = (top_n is not None and top_n <= 10)
501
  fig.update_layout(
502
  title=None,
503
- hovermode='closest', # Show individual agent info on hover
504
  barmode='group',
505
  height=600,
506
  showlegend=show_legend,
@@ -516,10 +516,10 @@ def create_discussion_monthly_metrics_plot(top_n=5):
516
  - Left y-axis: Discussion Resolved Rate (%) as line curves
517
  - Right y-axis: Total Discussions created as bar charts
518
 
519
- Each agent gets a unique color for both their line and bars.
520
 
521
  Args:
522
- top_n: Number of top agents to show (default: 5)
523
  """
524
  # Load from saved dataset
525
  saved_data = load_leaderboard_data_from_hf()
@@ -544,10 +544,10 @@ def create_discussion_monthly_metrics_plot(top_n=5):
544
  print(f"Loaded discussion monthly metrics from saved dataset")
545
 
546
  # Apply top_n filter if specified
547
- if top_n is not None and top_n > 0 and metrics.get('agents'):
548
- # Calculate total discussions for each agent
549
  agent_totals = []
550
- for agent_name in metrics['agents']:
551
  agent_data = metrics['data'].get(agent_name, {})
552
  total_discussions = agent_data.get('total_discussions')
553
  agent_totals.append((agent_name, total_discussions))
@@ -556,14 +556,14 @@ def create_discussion_monthly_metrics_plot(top_n=5):
556
  agent_totals.sort(key=lambda x: x[1], reverse=True)
557
  top_agents = [agent_name for agent_name, _ in agent_totals[:top_n]]
558
 
559
- # Filter metrics to only include top agents
560
  metrics = {
561
- 'agents': top_agents,
562
  'months': metrics['months'],
563
- 'data': {agent: metrics['data'][agent] for agent in top_agents if agent in metrics['data']}
564
  }
565
 
566
- if not metrics['agents'] or not metrics['months']:
567
  # Return an empty figure with a message
568
  fig = go.Figure()
569
  fig.add_annotation(
@@ -582,7 +582,7 @@ def create_discussion_monthly_metrics_plot(top_n=5):
582
  # Create figure with secondary y-axis
583
  fig = make_subplots(specs=[[{"secondary_y": True}]])
584
 
585
- # Generate unique colors for many agents using HSL color space
586
  def generate_color(index, total):
587
  """Generate distinct colors using HSL color space for better distribution"""
588
  hue = (index * 360 / total) % 360
@@ -590,15 +590,15 @@ def create_discussion_monthly_metrics_plot(top_n=5):
590
  lightness = 45 + (index % 2) * 10 # Vary lightness slightly
591
  return f'hsl({hue}, {saturation}%, {lightness}%)'
592
 
593
- agents = metrics['agents']
594
  months = metrics['months']
595
  data = metrics['data']
596
 
597
- # Generate colors for all agents
598
- agent_colors = {agent: generate_color(idx, len(agents)) for idx, agent in enumerate(agents)}
599
 
600
- # Add traces for each agent
601
- for idx, agent_name in enumerate(agents):
602
  color = agent_colors[agent_name]
603
  agent_data = data[agent_name]
604
 
@@ -618,8 +618,8 @@ def create_discussion_monthly_metrics_plot(top_n=5):
618
  line=dict(color=color, width=2),
619
  marker=dict(size=8),
620
  legendgroup=agent_name,
621
- showlegend=(top_n is not None and top_n <= 10), # Show legend for top N agents
622
- hovertemplate='<b>Agent: %{fullData.name}</b><br>' +
623
  'Month: %{x}<br>' +
624
  'Discussion Resolved Rate: %{y:.2f}%<br>' +
625
  '<extra></extra>'
@@ -628,7 +628,7 @@ def create_discussion_monthly_metrics_plot(top_n=5):
628
  )
629
 
630
  # Add bar trace for total discussions (right y-axis)
631
- # Only show bars for months where agent has discussions
632
  x_bars = []
633
  y_bars = []
634
  for month, count in zip(months, agent_data['total_discussions']):
@@ -645,11 +645,11 @@ def create_discussion_monthly_metrics_plot(top_n=5):
645
  marker=dict(color=color, opacity=0.6),
646
  legendgroup=agent_name,
647
  showlegend=False, # Hide duplicate legend entry (already shown in Scatter)
648
- hovertemplate='<b>Agent: %{fullData.name}</b><br>' +
649
  'Month: %{x}<br>' +
650
  'Total Discussions: %{y}<br>' +
651
  '<extra></extra>',
652
- offsetgroup=agent_name # Group bars by agent for proper spacing
653
  ),
654
  secondary_y=True
655
  )
@@ -671,7 +671,7 @@ def create_discussion_monthly_metrics_plot(top_n=5):
671
  show_legend = (top_n is not None and top_n <= 10)
672
  fig.update_layout(
673
  title=None,
674
- hovermode='closest', # Show individual agent info on hover
675
  barmode='group',
676
  height=600,
677
  showlegend=show_legend,
@@ -710,9 +710,9 @@ def get_leaderboard_dataframe():
710
  filtered_count = 0
711
  for identifier, data in cache_dict.items():
712
  total_issues = data.get('total_issues', 0)
713
- print(f" Agent '{identifier}': {total_issues} issues")
714
 
715
- # Filter out agents with zero total issues
716
  if total_issues == 0:
717
  filtered_count += 1
718
  continue
@@ -730,8 +730,8 @@ def get_leaderboard_dataframe():
730
  data.get('resolved_discussions', 0), # Resolved Discussions
731
  ])
732
 
733
- print(f"Filtered out {filtered_count} agents with 0 issues")
734
- print(f"Leaderboard will show {len(rows)} agents")
735
 
736
  # Create DataFrame
737
  column_names = [col[0] for col in LEADERBOARD_COLUMNS]
@@ -807,14 +807,14 @@ def get_wanted_issues_dataframe():
807
 
808
  def submit_agent(identifier, agent_name, organization, website):
809
  """
810
- Submit a new agent to the leaderboard.
811
  Validates input and saves submission.
812
  """
813
  # Validate required fields
814
  if not identifier or not identifier.strip():
815
  return "ERROR: GitHub identifier is required", gr.update()
816
  if not agent_name or not agent_name.strip():
817
- return "ERROR: Agent name is required", gr.update()
818
  if not organization or not organization.strip():
819
  return "ERROR: Organization name is required", gr.update()
820
  if not website or not website.strip():
@@ -831,12 +831,12 @@ def submit_agent(identifier, agent_name, organization, website):
831
  if not is_valid:
832
  return f"ERROR: {message}", gr.update()
833
 
834
- # Check for duplicates by loading agents from HuggingFace
835
- agents = load_agents_from_hf()
836
- if agents:
837
- existing_names = {agent['github_identifier'] for agent in agents}
838
  if identifier in existing_names:
839
- return f"WARNING: Agent with identifier '{identifier}' already exists", gr.update()
840
 
841
  # Create submission
842
  submission = {
@@ -873,7 +873,7 @@ def reload_leaderboard_data():
873
  if data:
874
  print(f"Successfully reloaded leaderboard data")
875
  print(f" Last updated: {data.get('metadata', {}).get('last_updated', 'Unknown')}")
876
- print(f" Agents: {len(data.get('leaderboard', {}))}")
877
  else:
878
  print(f"No data available")
879
  except Exception as e:
@@ -886,7 +886,7 @@ def reload_leaderboard_data():
886
  # GRADIO APPLICATION
887
  # =============================================================================
888
 
889
- print(f"\nStarting SWE Agent Issue Leaderboard")
890
  print(f" Data source: {LEADERBOARD_REPO}")
891
  print(f" Reload frequency: Daily at 12:00 AM UTC\n")
892
 
@@ -907,19 +907,19 @@ print(f"On startup: Loads cached data from HuggingFace on demand")
907
  print(f"{'='*80}\n")
908
 
909
  # Create Gradio interface
910
- with gr.Blocks(title="SWE Agent Issue & Discussion Leaderboard", theme=gr.themes.Soft()) as app:
911
- gr.Markdown("# SWE Agent Issue & Discussion Leaderboard")
912
- gr.Markdown(f"Track and compare GitHub issue and discussion resolution statistics for SWE agents")
913
 
914
  with gr.Tabs():
915
 
916
  # Leaderboard Tab
917
  with gr.Tab("Leaderboard"):
918
- gr.Markdown("*Statistics are based on agent issue resolution activity tracked by the system*")
919
  leaderboard_table = Leaderboard(
920
  value=pd.DataFrame(columns=[col[0] for col in LEADERBOARD_COLUMNS]), # Empty initially
921
  datatype=LEADERBOARD_COLUMNS,
922
- search_columns=["Agent Name", "Website"],
923
  filter_columns=[
924
  ColumnFilter(
925
  "Issue Resolved Rate (%)",
@@ -942,8 +942,8 @@ with gr.Blocks(title="SWE Agent Issue & Discussion Leaderboard", theme=gr.themes
942
  # Monthly Metrics Section
943
  gr.Markdown("---") # Divider
944
  with gr.Group():
945
- gr.Markdown("### Issue Monthly Performance - Top 5 Agents")
946
- gr.Markdown("*Shows issue resolution trends and volumes for the most active agents*")
947
  monthly_metrics_plot = gr.Plot(label="Issue Monthly Metrics")
948
 
949
  # Load monthly metrics when app starts
@@ -956,8 +956,8 @@ with gr.Blocks(title="SWE Agent Issue & Discussion Leaderboard", theme=gr.themes
956
  # Discussion Monthly Metrics Section
957
  gr.Markdown("---") # Divider
958
  with gr.Group():
959
- gr.Markdown("### Discussion Monthly Performance - Top 5 Agents")
960
- gr.Markdown("*Shows discussion resolution trends and volumes for the most active agents*")
961
  discussion_metrics_plot = gr.Plot(label="Discussion Monthly Metrics")
962
 
963
  # Load discussion monthly metrics when app starts
@@ -987,20 +987,20 @@ with gr.Blocks(title="SWE Agent Issue & Discussion Leaderboard", theme=gr.themes
987
  )
988
 
989
 
990
- # Submit Agent Tab
991
- with gr.Tab("Submit Your Agent"):
992
 
993
- gr.Markdown("Fill in the details below to add your agent to the leaderboard.")
994
 
995
  with gr.Row():
996
  with gr.Column():
997
  github_input = gr.Textbox(
998
  label="GitHub Identifier*",
999
- placeholder="Your agent username (e.g., my-agent[bot])"
1000
  )
1001
  name_input = gr.Textbox(
1002
- label="Agent Name*",
1003
- placeholder="Your agent's display name"
1004
  )
1005
 
1006
  with gr.Column():
@@ -1010,11 +1010,11 @@ with gr.Blocks(title="SWE Agent Issue & Discussion Leaderboard", theme=gr.themes
1010
  )
1011
  website_input = gr.Textbox(
1012
  label="Website*",
1013
- placeholder="https://your-agent-website.com"
1014
  )
1015
 
1016
  submit_button = gr.Button(
1017
- "Submit Agent",
1018
  variant="primary"
1019
  )
1020
  submission_status = gr.Textbox(
 
24
  # CONFIGURATION
25
  # =============================================================================
26
 
27
+ AGENTS_REPO = "SWE-Arena/bot_metadata" # HuggingFace dataset for assistant metadata
28
  AGENTS_REPO_LOCAL_PATH = os.path.expanduser("~/bot_metadata") # Local git clone path
29
  LEADERBOARD_FILENAME = f"{os.getenv('COMPOSE_PROJECT_NAME')}.json"
30
  LEADERBOARD_REPO = "SWE-Arena/leaderboard_data" # HuggingFace dataset for leaderboard data
 
33
  MAX_RETRIES = 5
34
 
35
  LEADERBOARD_COLUMNS = [
36
+ ("Assistant", "string"),
37
  ("Website", "string"),
38
  ("Total Issues", "number"),
39
  ("Total Discussions", "number"),
 
162
 
163
  def load_agents_from_hf():
164
  """
165
+ Load all assistant metadata JSON files from local git repository.
166
  ALWAYS syncs with remote first to ensure we have the latest bot data.
167
  """
168
  # MANDATORY: Sync with remote first to get latest bot data
169
+ print(f" Syncing bot_metadata repository to get latest assistants...")
170
  sync_agents_repo() # Will raise exception if sync fails
171
 
172
+ assistants = []
173
 
174
  # Scan local directory for JSON files
175
  if not os.path.exists(AGENTS_REPO_LOCAL_PATH):
 
177
 
178
  # Walk through the directory to find all JSON files
179
  files_processed = 0
180
+ print(f" Loading assistant metadata from {AGENTS_REPO_LOCAL_PATH}...")
181
 
182
  for root, dirs, files in os.walk(AGENTS_REPO_LOCAL_PATH):
183
  # Skip .git directory
 
195
  with open(file_path, 'r', encoding='utf-8') as f:
196
  agent_data = json.load(f)
197
 
198
+ # Only include active assistants
199
  if agent_data.get('status') != 'active':
200
  continue
201
 
 
203
  github_identifier = filename.replace('.json', '')
204
  agent_data['github_identifier'] = github_identifier
205
 
206
+ assistants.append(agent_data)
207
 
208
  except Exception as e:
209
  print(f" Warning Error loading {filename}: {str(e)}")
210
  continue
211
 
212
+ print(f" Success Loaded {len(assistants)} active assistants (from {files_processed} total files)")
213
+ return assistants
214
 
215
 
216
  def get_hf_token():
 
265
 
266
 
267
  def save_agent_to_hf(data):
268
+ """Save a new assistant to HuggingFace dataset as {identifier}.json in root."""
269
  try:
270
  api = HfApi()
271
  token = get_hf_token()
 
290
  repo_type="dataset",
291
  token=token
292
  )
293
+ print(f"Saved assistant to HuggingFace: {filename}")
294
  return True
295
  finally:
296
  # Always clean up local file, even if upload fails
 
298
  os.remove(filename)
299
 
300
  except Exception as e:
301
+ print(f"Error saving assistant: {str(e)}")
302
  return False
303
 
304
 
 
345
  - Left y-axis: Resolved Rate (%) as line curves
346
  - Right y-axis: Total Issues created as bar charts
347
 
348
+ Each assistant gets a unique color for both their line and bars.
349
 
350
  Args:
351
+ top_n: Number of top assistants to show (default: 5)
352
  """
353
  # Load from saved dataset
354
  saved_data = load_leaderboard_data_from_hf()
 
373
  print(f"Loaded monthly metrics from saved dataset")
374
 
375
  # Apply top_n filter if specified
376
+ if top_n is not None and top_n > 0 and metrics.get('assistants'):
377
+ # Calculate total issues for each assistant
378
  agent_totals = []
379
+ for agent_name in metrics['assistants']:
380
  agent_data = metrics['data'].get(agent_name, {})
381
  total_issues = sum(agent_data.get('total_issues', []))
382
  agent_totals.append((agent_name, total_issues))
 
385
  agent_totals.sort(key=lambda x: x[1], reverse=True)
386
  top_agents = [agent_name for agent_name, _ in agent_totals[:top_n]]
387
 
388
+ # Filter metrics to only include top assistants
389
  metrics = {
390
+ 'assistants': top_agents,
391
  'months': metrics['months'],
392
+ 'data': {assistant: metrics['data'][assistant] for assistant in top_agents if assistant in metrics['data']}
393
  }
394
 
395
+ if not metrics['assistants'] or not metrics['months']:
396
  # Return an empty figure with a message
397
  fig = go.Figure()
398
  fig.add_annotation(
 
411
  # Create figure with secondary y-axis
412
  fig = make_subplots(specs=[[{"secondary_y": True}]])
413
 
414
+ # Generate unique colors for many assistants using HSL color space
415
  def generate_color(index, total):
416
  """Generate distinct colors using HSL color space for better distribution"""
417
  hue = (index * 360 / total) % 360
 
419
  lightness = 45 + (index % 2) * 10 # Vary lightness slightly
420
  return f'hsl({hue}, {saturation}%, {lightness}%)'
421
 
422
+ assistants = metrics['assistants']
423
  months = metrics['months']
424
  data = metrics['data']
425
 
426
+ # Generate colors for all assistants
427
+ agent_colors = {assistant: generate_color(idx, len(assistants)) for idx, assistant in enumerate(assistants)}
428
 
429
+ # Add traces for each assistant
430
+ for idx, agent_name in enumerate(assistants):
431
  color = agent_colors[agent_name]
432
  agent_data = data[agent_name]
433
 
 
447
  line=dict(color=color, width=2),
448
  marker=dict(size=8),
449
  legendgroup=agent_name,
450
+ showlegend=(top_n is not None and top_n <= 10), # Show legend for top N assistants
451
+ hovertemplate='<b>Assistant: %{fullData.name}</b><br>' +
452
  'Month: %{x}<br>' +
453
  'Resolved Rate: %{y:.2f}%<br>' +
454
  '<extra></extra>'
 
457
  )
458
 
459
  # Add bar trace for total issues (right y-axis)
460
+ # Only show bars for months where assistant has issues
461
  x_bars = []
462
  y_bars = []
463
  for month, count in zip(months, agent_data['total_issues']):
 
474
  marker=dict(color=color, opacity=0.6),
475
  legendgroup=agent_name,
476
  showlegend=False, # Hide duplicate legend entry (already shown in Scatter)
477
+ hovertemplate='<b>Assistant: %{fullData.name}</b><br>' +
478
  'Month: %{x}<br>' +
479
  'Total Issues: %{y}<br>' +
480
  '<extra></extra>',
481
+ offsetgroup=agent_name # Group bars by assistant for proper spacing
482
  ),
483
  secondary_y=True
484
  )
 
500
  show_legend = (top_n is not None and top_n <= 10)
501
  fig.update_layout(
502
  title=None,
503
+ hovermode='closest', # Show individual assistant info on hover
504
  barmode='group',
505
  height=600,
506
  showlegend=show_legend,
 
516
  - Left y-axis: Discussion Resolved Rate (%) as line curves
517
  - Right y-axis: Total Discussions created as bar charts
518
 
519
+ Each assistant gets a unique color for both their line and bars.
520
 
521
  Args:
522
+ top_n: Number of top assistants to show (default: 5)
523
  """
524
  # Load from saved dataset
525
  saved_data = load_leaderboard_data_from_hf()
 
544
  print(f"Loaded discussion monthly metrics from saved dataset")
545
 
546
  # Apply top_n filter if specified
547
+ if top_n is not None and top_n > 0 and metrics.get('assistants'):
548
+ # Calculate total discussions for each assistant
549
  agent_totals = []
550
+ for agent_name in metrics['assistants']:
551
  agent_data = metrics['data'].get(agent_name, {})
552
  total_discussions = agent_data.get('total_discussions')
553
  agent_totals.append((agent_name, total_discussions))
 
556
  agent_totals.sort(key=lambda x: x[1], reverse=True)
557
  top_agents = [agent_name for agent_name, _ in agent_totals[:top_n]]
558
 
559
+ # Filter metrics to only include top assistants
560
  metrics = {
561
+ 'assistants': top_agents,
562
  'months': metrics['months'],
563
+ 'data': {assistant: metrics['data'][assistant] for assistant in top_agents if assistant in metrics['data']}
564
  }
565
 
566
+ if not metrics['assistants'] or not metrics['months']:
567
  # Return an empty figure with a message
568
  fig = go.Figure()
569
  fig.add_annotation(
 
582
  # Create figure with secondary y-axis
583
  fig = make_subplots(specs=[[{"secondary_y": True}]])
584
 
585
+ # Generate unique colors for many assistants using HSL color space
586
  def generate_color(index, total):
587
  """Generate distinct colors using HSL color space for better distribution"""
588
  hue = (index * 360 / total) % 360
 
590
  lightness = 45 + (index % 2) * 10 # Vary lightness slightly
591
  return f'hsl({hue}, {saturation}%, {lightness}%)'
592
 
593
+ assistants = metrics['assistants']
594
  months = metrics['months']
595
  data = metrics['data']
596
 
597
+ # Generate colors for all assistants
598
+ agent_colors = {assistant: generate_color(idx, len(assistants)) for idx, assistant in enumerate(assistants)}
599
 
600
+ # Add traces for each assistant
601
+ for idx, agent_name in enumerate(assistants):
602
  color = agent_colors[agent_name]
603
  agent_data = data[agent_name]
604
 
 
618
  line=dict(color=color, width=2),
619
  marker=dict(size=8),
620
  legendgroup=agent_name,
621
+ showlegend=(top_n is not None and top_n <= 10), # Show legend for top N assistants
622
+ hovertemplate='<b>Assistant: %{fullData.name}</b><br>' +
623
  'Month: %{x}<br>' +
624
  'Discussion Resolved Rate: %{y:.2f}%<br>' +
625
  '<extra></extra>'
 
628
  )
629
 
630
  # Add bar trace for total discussions (right y-axis)
631
+ # Only show bars for months where assistant has discussions
632
  x_bars = []
633
  y_bars = []
634
  for month, count in zip(months, agent_data['total_discussions']):
 
645
  marker=dict(color=color, opacity=0.6),
646
  legendgroup=agent_name,
647
  showlegend=False, # Hide duplicate legend entry (already shown in Scatter)
648
+ hovertemplate='<b>Assistant: %{fullData.name}</b><br>' +
649
  'Month: %{x}<br>' +
650
  'Total Discussions: %{y}<br>' +
651
  '<extra></extra>',
652
+ offsetgroup=agent_name # Group bars by assistant for proper spacing
653
  ),
654
  secondary_y=True
655
  )
 
671
  show_legend = (top_n is not None and top_n <= 10)
672
  fig.update_layout(
673
  title=None,
674
+ hovermode='closest', # Show individual assistant info on hover
675
  barmode='group',
676
  height=600,
677
  showlegend=show_legend,
 
710
  filtered_count = 0
711
  for identifier, data in cache_dict.items():
712
  total_issues = data.get('total_issues', 0)
713
+ print(f" Assistant '{identifier}': {total_issues} issues")
714
 
715
+ # Filter out assistants with zero total issues
716
  if total_issues == 0:
717
  filtered_count += 1
718
  continue
 
730
  data.get('resolved_discussions', 0), # Resolved Discussions
731
  ])
732
 
733
+ print(f"Filtered out {filtered_count} assistants with 0 issues")
734
+ print(f"Leaderboard will show {len(rows)} assistants")
735
 
736
  # Create DataFrame
737
  column_names = [col[0] for col in LEADERBOARD_COLUMNS]
 
807
 
808
  def submit_agent(identifier, agent_name, organization, website):
809
  """
810
+ Submit a new assistant to the leaderboard.
811
  Validates input and saves submission.
812
  """
813
  # Validate required fields
814
  if not identifier or not identifier.strip():
815
  return "ERROR: GitHub identifier is required", gr.update()
816
  if not agent_name or not agent_name.strip():
817
+ return "ERROR: Assistant name is required", gr.update()
818
  if not organization or not organization.strip():
819
  return "ERROR: Organization name is required", gr.update()
820
  if not website or not website.strip():
 
831
  if not is_valid:
832
  return f"ERROR: {message}", gr.update()
833
 
834
+ # Check for duplicates by loading assistants from HuggingFace
835
+ assistants = load_agents_from_hf()
836
+ if assistants:
837
+ existing_names = {assistant['github_identifier'] for assistant in assistants}
838
  if identifier in existing_names:
839
+ return f"WARNING: Assistant with identifier '{identifier}' already exists", gr.update()
840
 
841
  # Create submission
842
  submission = {
 
873
  if data:
874
  print(f"Successfully reloaded leaderboard data")
875
  print(f" Last updated: {data.get('metadata', {}).get('last_updated', 'Unknown')}")
876
+ print(f" Assistants: {len(data.get('leaderboard', {}))}")
877
  else:
878
  print(f"No data available")
879
  except Exception as e:
 
886
  # GRADIO APPLICATION
887
  # =============================================================================
888
 
889
+ print(f"\nStarting SWE Assistant Issue Leaderboard")
890
  print(f" Data source: {LEADERBOARD_REPO}")
891
  print(f" Reload frequency: Daily at 12:00 AM UTC\n")
892
 
 
907
  print(f"{'='*80}\n")
908
 
909
  # Create Gradio interface
910
+ with gr.Blocks(title="SWE Assistant Issue & Discussion Leaderboard", theme=gr.themes.Soft()) as app:
911
+ gr.Markdown("# SWE Assistant Issue & Discussion Leaderboard")
912
+ gr.Markdown(f"Track and compare GitHub issue and discussion resolution statistics for SWE assistants")
913
 
914
  with gr.Tabs():
915
 
916
  # Leaderboard Tab
917
  with gr.Tab("Leaderboard"):
918
+ gr.Markdown("*Statistics are based on assistant issue resolution activity tracked by the system*")
919
  leaderboard_table = Leaderboard(
920
  value=pd.DataFrame(columns=[col[0] for col in LEADERBOARD_COLUMNS]), # Empty initially
921
  datatype=LEADERBOARD_COLUMNS,
922
+ search_columns=["Assistant", "Website"],
923
  filter_columns=[
924
  ColumnFilter(
925
  "Issue Resolved Rate (%)",
 
942
  # Monthly Metrics Section
943
  gr.Markdown("---") # Divider
944
  with gr.Group():
945
+ gr.Markdown("### Issue Monthly Performance - Top 5 Assistants")
946
+ gr.Markdown("*Shows issue resolution trends and volumes for the most active assistants*")
947
  monthly_metrics_plot = gr.Plot(label="Issue Monthly Metrics")
948
 
949
  # Load monthly metrics when app starts
 
956
  # Discussion Monthly Metrics Section
957
  gr.Markdown("---") # Divider
958
  with gr.Group():
959
+ gr.Markdown("### Discussion Monthly Performance - Top 5 Assistants")
960
+ gr.Markdown("*Shows discussion resolution trends and volumes for the most active assistants*")
961
  discussion_metrics_plot = gr.Plot(label="Discussion Monthly Metrics")
962
 
963
  # Load discussion monthly metrics when app starts
 
987
  )
988
 
989
 
990
+ # Submit Assistant Tab
991
+ with gr.Tab("Submit Your Assistant"):
992
 
993
+ gr.Markdown("Fill in the details below to add your assistant to the leaderboard.")
994
 
995
  with gr.Row():
996
  with gr.Column():
997
  github_input = gr.Textbox(
998
  label="GitHub Identifier*",
999
+ placeholder="Your assistant username (e.g., my-assistant[bot])"
1000
  )
1001
  name_input = gr.Textbox(
1002
+ label="Assistant Name*",
1003
+ placeholder="Your assistant's display name"
1004
  )
1005
 
1006
  with gr.Column():
 
1010
  )
1011
  website_input = gr.Textbox(
1012
  label="Website*",
1013
+ placeholder="https://your-assistant-website.com"
1014
  )
1015
 
1016
  submit_button = gr.Button(
1017
+ "Submit Assistant",
1018
  variant="primary"
1019
  )
1020
  submission_status = gr.Textbox(
msr.py CHANGED
@@ -361,14 +361,14 @@ def generate_file_path_patterns(start_date, end_date, data_dir=GHARCHIVE_DATA_LO
361
  def fetch_all_metadata_streaming(conn, identifiers, start_date, end_date):
362
  """
363
  UNIFIED QUERY: Fetches ALL metadata types in ONE query per batch:
364
- - IssuesEvent, IssueCommentEvent (for agent-assigned issues AND wanted issues)
365
  - PullRequestEvent (for wanted issue tracking)
366
  - DiscussionEvent (for discussion tracking)
367
 
368
  Then post-processes in Python to separate into:
369
- 1. Agent-assigned issues: Issues where agents are assigned to or commented on
370
- 2. Wanted issues: Long-standing issues from tracked orgs linked to merged PRs by agents
371
- 3. Discussions: GitHub discussions created by agents
372
 
373
  This approach is more efficient than running separate queries for each category.
374
 
@@ -380,17 +380,17 @@ def fetch_all_metadata_streaming(conn, identifiers, start_date, end_date):
380
 
381
  Returns:
382
  Dictionary with four keys:
383
- - 'agent_issues': {agent_id: [issue_metadata]} for agent-assigned issues
384
  - 'wanted_open': [open_wanted_issues] for long-standing open issues
385
  - 'wanted_resolved': {agent_id: [resolved_wanted]} for resolved wanted issues
386
- - 'agent_discussions': {agent_id: [discussion_metadata]} for agent discussions
387
  """
388
  print(f" Fetching ALL metadata (issues, PRs, discussions) with unified query...")
389
  identifier_set = set(identifiers)
390
  identifier_list = ', '.join([f"'{id}'" for id in identifiers])
391
  tracked_orgs_list = ', '.join([f"'{org}'" for org in TRACKED_ORGS])
392
 
393
- # Storage for agent-assigned issues
394
  agent_issues = defaultdict(list) # agent_id -> [issue_metadata]
395
  agent_issue_urls = defaultdict(set) # agent_id -> set of issue URLs (for deduplication)
396
 
@@ -433,7 +433,7 @@ def fetch_all_metadata_streaming(conn, identifiers, start_date, end_date):
433
 
434
  try:
435
  # UNIFIED QUERY: Fetch ALL event types in ONE query
436
- # Post-process in Python to separate into agent-assigned issues, wanted issues, PRs, and discussions
437
  unified_query = f"""
438
  SELECT
439
  type,
@@ -448,7 +448,7 @@ def fetch_all_metadata_streaming(conn, identifiers, start_date, end_date):
448
  json_extract(payload, '$.issue.labels') as issue_labels,
449
  json_extract_string(payload, '$.issue.pull_request') as is_pull_request,
450
  json_extract_string(payload, '$.issue.state_reason') as issue_state_reason,
451
- -- Actor/assignee fields for agent assignment
452
  json_extract_string(payload, '$.issue.user.login') as issue_creator,
453
  json_extract_string(payload, '$.issue.assignee.login') as issue_assignee,
454
  json_extract(payload, '$.issue.assignees') as issue_assignees,
@@ -481,7 +481,7 @@ def fetch_all_metadata_streaming(conn, identifiers, start_date, end_date):
481
  WHERE
482
  type IN ('IssuesEvent', 'IssueCommentEvent', 'PullRequestEvent', 'DiscussionEvent')
483
  AND (
484
- -- Agent-assigned issues: agent is creator, assignee, or commenter
485
  (type = 'IssuesEvent' AND (
486
  json_extract_string(payload, '$.issue.user.login') IN ({identifier_list})
487
  OR json_extract_string(payload, '$.issue.assignee.login') IN ({identifier_list})
@@ -491,17 +491,17 @@ def fetch_all_metadata_streaming(conn, identifiers, start_date, end_date):
491
  )
492
  OR SPLIT_PART(json_extract_string(repo, '$.name'), '/', 1) IN ({tracked_orgs_list})
493
  ))
494
- -- Issue comments: agent is commenter OR tracked org
495
  OR (type = 'IssueCommentEvent' AND (
496
  json_extract_string(payload, '$.comment.user.login') IN ({identifier_list})
497
  OR SPLIT_PART(json_extract_string(repo, '$.name'), '/', 1) IN ({tracked_orgs_list})
498
  ))
499
- -- PRs: agent is creator OR tracked org (for wanted issue tracking)
500
  OR (type = 'PullRequestEvent' AND (
501
  json_extract_string(payload, '$.pull_request.user.login') IN ({identifier_list})
502
  OR SPLIT_PART(json_extract_string(repo, '$.name'), '/', 1) IN ({tracked_orgs_list})
503
  ))
504
- -- Discussions: agent is creator AND tracked org
505
  OR (type = 'DiscussionEvent'
506
  AND json_extract_string(payload, '$.discussion.user.login') IN ({identifier_list})
507
  AND SPLIT_PART(json_extract_string(repo, '$.name'), '/', 1) IN ({tracked_orgs_list})
@@ -522,7 +522,7 @@ def fetch_all_metadata_streaming(conn, identifiers, start_date, end_date):
522
  issue_events = [] # For wanted tracking
523
  pr_events = [] # For wanted tracking
524
  discussion_events = [] # For discussion tracking
525
- agent_issue_events = [] # For agent-assigned issues
526
 
527
  for row in all_results:
528
  event_type = row[0]
@@ -530,7 +530,7 @@ def fetch_all_metadata_streaming(conn, identifiers, start_date, end_date):
530
 
531
  if event_type in ('IssuesEvent', 'IssueCommentEvent'):
532
  if not is_pr: # It's an issue, not a PR
533
- # Check if this is an agent-assigned issue
534
  issue_creator = row[11]
535
  issue_assignee = row[12]
536
  issue_assignees_json = row[13]
@@ -564,7 +564,7 @@ def fetch_all_metadata_streaming(conn, identifiers, start_date, end_date):
564
  pass
565
 
566
  elif event_type == 'IssueCommentEvent':
567
- # Check if commenter is an agent
568
  if commenter in identifier_set:
569
  agent_identifier = commenter
570
 
@@ -584,7 +584,7 @@ def fetch_all_metadata_streaming(conn, identifiers, start_date, end_date):
584
  elif event_type == 'DiscussionEvent':
585
  discussion_events.append(row)
586
 
587
- # Process agent-assigned issues
588
  for row, agent_identifier in agent_issue_events:
589
  # Row indices: repo_url=2, issue_url=3, issue_created_at=6, issue_closed_at=7, issue_state_reason=10
590
  repo_url = row[2]
@@ -612,7 +612,7 @@ def fetch_all_metadata_streaming(conn, identifiers, start_date, end_date):
612
  except:
613
  continue
614
 
615
- # Deduplicate: only add if we haven't seen this issue for this agent
616
  if full_url in agent_issue_urls[agent_identifier]:
617
  continue
618
 
@@ -753,8 +753,8 @@ def fetch_all_metadata_streaming(conn, identifiers, start_date, end_date):
753
  except:
754
  continue
755
 
756
- # Group by creator (agent identifier)
757
- # Only track discussions from our agent identifiers
758
  if discussion_creator not in identifier_set:
759
  continue
760
 
@@ -776,12 +776,12 @@ def fetch_all_metadata_streaming(conn, identifiers, start_date, end_date):
776
  'state_reason': discussion_state_reason
777
  }
778
 
779
- # Group by agent
780
  if discussion_creator not in discussions_by_agent:
781
  discussions_by_agent[discussion_creator] = []
782
  discussions_by_agent[discussion_creator].append(discussion_meta)
783
 
784
- print(f"✓ {len(agent_issue_events)} agent issues, {len(issue_events)} wanted issues, {len(pr_events)} PRs, {len(discussion_events)} discussions")
785
 
786
  except Exception as e:
787
  print(f"\n ✗ Batch {batch_num} error: {str(e)}")
@@ -790,7 +790,7 @@ def fetch_all_metadata_streaming(conn, identifiers, start_date, end_date):
790
  # Move to next batch
791
  current_date = batch_end + timedelta(days=1)
792
 
793
- # Post-processing: Filter issues and assign to agents
794
  print(f"\n Post-processing {len(all_issues)} wanted issues...")
795
 
796
  wanted_open = []
@@ -803,7 +803,7 @@ def fetch_all_metadata_streaming(conn, identifiers, start_date, end_date):
803
  if not linked_prs:
804
  continue
805
 
806
- # Check if any linked PR was merged AND created by an agent
807
  resolved_by = None
808
  for pr_url in linked_prs:
809
  merged_at = pr_merged_at.get(pr_url)
@@ -862,10 +862,10 @@ def fetch_all_metadata_streaming(conn, identifiers, start_date, end_date):
862
  except:
863
  pass
864
 
865
- print(f" ✓ Found {sum(len(issues) for issues in agent_issues.values())} agent-assigned issues across {len(agent_issues)} agents")
866
  print(f" ✓ Found {len(wanted_open)} long-standing open wanted issues")
867
- print(f" ✓ Found {sum(len(issues) for issues in wanted_resolved.values())} resolved wanted issues across {len(wanted_resolved)} agents")
868
- print(f" ✓ Found {sum(len(discussions) for discussions in discussions_by_agent.values())} discussions across {len(discussions_by_agent)} agents")
869
 
870
  return {
871
  'agent_issues': dict(agent_issues),
@@ -933,14 +933,14 @@ def sync_agents_repo():
933
 
934
  def load_agents_from_hf():
935
  """
936
- Load all agent metadata JSON files from local git repository.
937
  ALWAYS syncs with remote first to ensure we have the latest bot data.
938
  """
939
  # MANDATORY: Sync with remote first to get latest bot data
940
- print(f" Syncing bot_metadata repository to get latest agents...")
941
  sync_agents_repo() # Will raise exception if sync fails
942
 
943
- agents = []
944
 
945
  # Scan local directory for JSON files
946
  if not os.path.exists(AGENTS_REPO_LOCAL_PATH):
@@ -948,7 +948,7 @@ def load_agents_from_hf():
948
 
949
  # Walk through the directory to find all JSON files
950
  files_processed = 0
951
- print(f" Loading agent metadata from {AGENTS_REPO_LOCAL_PATH}...")
952
 
953
  for root, dirs, files in os.walk(AGENTS_REPO_LOCAL_PATH):
954
  # Skip .git directory
@@ -966,7 +966,7 @@ def load_agents_from_hf():
966
  with open(file_path, 'r', encoding='utf-8') as f:
967
  agent_data = json.load(f)
968
 
969
- # Only include active agents
970
  if agent_data.get('status') != 'active':
971
  continue
972
 
@@ -974,14 +974,14 @@ def load_agents_from_hf():
974
  github_identifier = filename.replace('.json', '')
975
  agent_data['github_identifier'] = github_identifier
976
 
977
- agents.append(agent_data)
978
 
979
  except Exception as e:
980
  print(f" ⚠ Error loading {filename}: {str(e)}")
981
  continue
982
 
983
- print(f" ✓ Loaded {len(agents)} active agents (from {files_processed} total files)")
984
- return agents
985
 
986
 
987
  def calculate_issue_stats_from_metadata(metadata_list):
@@ -1002,12 +1002,12 @@ def calculate_issue_stats_from_metadata(metadata_list):
1002
  }
1003
 
1004
 
1005
- def calculate_monthly_metrics_by_agent(all_metadata_dict, agents):
1006
- """Calculate monthly metrics for all agents for visualization."""
1007
- identifier_to_name = {agent.get('github_identifier'): agent.get('name') for agent in agents if agent.get('github_identifier')}
1008
 
1009
  if not all_metadata_dict:
1010
- return {'agents': [], 'months': [], 'data': {}}
1011
 
1012
  agent_month_data = defaultdict(lambda: defaultdict(list))
1013
 
@@ -1065,7 +1065,7 @@ def calculate_monthly_metrics_by_agent(all_metadata_dict, agents):
1065
  agents_list = sorted(list(agent_month_data.keys()))
1066
 
1067
  return {
1068
- 'agents': agents_list,
1069
  'months': months,
1070
  'data': result_data
1071
  }
@@ -1086,12 +1086,12 @@ def calculate_discussion_stats_from_metadata(metadata_list):
1086
  }
1087
 
1088
 
1089
- def calculate_monthly_metrics_by_agent_discussions(all_discussions_dict, agents):
1090
- """Calculate monthly metrics for discussions for all agents for visualization."""
1091
- identifier_to_name = {agent.get('github_identifier'): agent.get('name') for agent in agents if agent.get('github_identifier')}
1092
 
1093
  if not all_discussions_dict:
1094
- return {'agents': [], 'months': [], 'data': {}}
1095
 
1096
  agent_month_data = defaultdict(lambda: defaultdict(list))
1097
 
@@ -1145,23 +1145,23 @@ def calculate_monthly_metrics_by_agent_discussions(all_discussions_dict, agents)
1145
  agents_list = sorted(list(agent_month_data.keys()))
1146
 
1147
  return {
1148
- 'agents': agents_list,
1149
  'months': months,
1150
  'data': result_data
1151
  }
1152
 
1153
 
1154
- def construct_leaderboard_from_metadata(all_metadata_dict, agents, wanted_resolved_dict=None, discussions_dict=None):
1155
  """Construct leaderboard from in-memory issue metadata and discussion metadata.
1156
 
1157
  Args:
1158
- all_metadata_dict: Dictionary mapping agent ID to list of issue metadata (agent-assigned issues)
1159
- agents: List of agent metadata
1160
- wanted_resolved_dict: Optional dictionary mapping agent ID to list of resolved wanted issues
1161
- discussions_dict: Optional dictionary mapping agent ID to list of discussion metadata
1162
  """
1163
- if not agents:
1164
- print("Error: No agents found")
1165
  return {}
1166
 
1167
  if wanted_resolved_dict is None:
@@ -1172,9 +1172,9 @@ def construct_leaderboard_from_metadata(all_metadata_dict, agents, wanted_resolv
1172
 
1173
  cache_dict = {}
1174
 
1175
- for agent in agents:
1176
- identifier = agent.get('github_identifier')
1177
- agent_name = agent.get('name', 'Unknown')
1178
 
1179
  bot_metadata = all_metadata_dict.get(identifier, [])
1180
  stats = calculate_issue_stats_from_metadata(bot_metadata)
@@ -1188,7 +1188,7 @@ def construct_leaderboard_from_metadata(all_metadata_dict, agents, wanted_resolv
1188
 
1189
  cache_dict[identifier] = {
1190
  'name': agent_name,
1191
- 'website': agent.get('website', 'N/A'),
1192
  'github_identifier': identifier,
1193
  **stats,
1194
  'resolved_wanted_issues': resolved_wanted,
@@ -1211,7 +1211,7 @@ def save_leaderboard_data_to_hf(leaderboard_dict, monthly_metrics, wanted_issues
1211
  wanted_issues = []
1212
 
1213
  if discussion_monthly_metrics is None:
1214
- discussion_monthly_metrics = {'agents': [], 'months': [], 'data': {}}
1215
 
1216
  combined_data = {
1217
  'metadata': {
@@ -1255,7 +1255,7 @@ def save_leaderboard_data_to_hf(leaderboard_dict, monthly_metrics, wanted_issues
1255
 
1256
  def mine_all_agents():
1257
  """
1258
- Mine issue metadata for all agents using STREAMING batch processing.
1259
  Downloads GHArchive data, then uses BATCH-based DuckDB queries.
1260
  """
1261
  print(f"\n[1/4] Downloading GHArchive data...")
@@ -1263,19 +1263,19 @@ def mine_all_agents():
1263
  if not download_all_gharchive_data():
1264
  print("Warning: Download had errors, continuing with available data...")
1265
 
1266
- print(f"\n[2/4] Loading agent metadata...")
1267
 
1268
- agents = load_agents_from_hf()
1269
- if not agents:
1270
- print("Error: No agents found")
1271
  return
1272
 
1273
- identifiers = [agent['github_identifier'] for agent in agents if agent.get('github_identifier')]
1274
  if not identifiers:
1275
- print("Error: No valid agent identifiers found")
1276
  return
1277
 
1278
- print(f"\n[3/4] Mining issue metadata ({len(identifiers)} agents, {LEADERBOARD_TIME_FRAME_DAYS} days)...")
1279
 
1280
  try:
1281
  conn = get_duckdb_connection()
@@ -1309,11 +1309,11 @@ def mine_all_agents():
1309
 
1310
  try:
1311
  leaderboard_dict = construct_leaderboard_from_metadata(
1312
- agent_issues, agents, wanted_resolved, agent_discussions
1313
  )
1314
- monthly_metrics = calculate_monthly_metrics_by_agent(agent_issues, agents)
1315
  discussion_monthly_metrics = calculate_monthly_metrics_by_agent_discussions(
1316
- agent_discussions, agents
1317
  )
1318
  save_leaderboard_data_to_hf(
1319
  leaderboard_dict, monthly_metrics, wanted_open, discussion_monthly_metrics
@@ -1350,7 +1350,7 @@ def setup_scheduler():
1350
  mine_all_agents,
1351
  trigger=trigger,
1352
  id='mine_all_agents',
1353
- name='Mine GHArchive data for all agents',
1354
  replace_existing=True
1355
  )
1356
 
 
361
  def fetch_all_metadata_streaming(conn, identifiers, start_date, end_date):
362
  """
363
  UNIFIED QUERY: Fetches ALL metadata types in ONE query per batch:
364
+ - IssuesEvent, IssueCommentEvent (for assistant-assigned issues AND wanted issues)
365
  - PullRequestEvent (for wanted issue tracking)
366
  - DiscussionEvent (for discussion tracking)
367
 
368
  Then post-processes in Python to separate into:
369
+ 1. Assistant-assigned issues: Issues where assistants are assigned to or commented on
370
+ 2. Wanted issues: Long-standing issues from tracked orgs linked to merged PRs by assistants
371
+ 3. Discussions: GitHub discussions created by assistants
372
 
373
  This approach is more efficient than running separate queries for each category.
374
 
 
380
 
381
  Returns:
382
  Dictionary with four keys:
383
+ - 'agent_issues': {agent_id: [issue_metadata]} for assistant-assigned issues
384
  - 'wanted_open': [open_wanted_issues] for long-standing open issues
385
  - 'wanted_resolved': {agent_id: [resolved_wanted]} for resolved wanted issues
386
+ - 'agent_discussions': {agent_id: [discussion_metadata]} for assistant discussions
387
  """
388
  print(f" Fetching ALL metadata (issues, PRs, discussions) with unified query...")
389
  identifier_set = set(identifiers)
390
  identifier_list = ', '.join([f"'{id}'" for id in identifiers])
391
  tracked_orgs_list = ', '.join([f"'{org}'" for org in TRACKED_ORGS])
392
 
393
+ # Storage for assistant-assigned issues
394
  agent_issues = defaultdict(list) # agent_id -> [issue_metadata]
395
  agent_issue_urls = defaultdict(set) # agent_id -> set of issue URLs (for deduplication)
396
 
 
433
 
434
  try:
435
  # UNIFIED QUERY: Fetch ALL event types in ONE query
436
+ # Post-process in Python to separate into assistant-assigned issues, wanted issues, PRs, and discussions
437
  unified_query = f"""
438
  SELECT
439
  type,
 
448
  json_extract(payload, '$.issue.labels') as issue_labels,
449
  json_extract_string(payload, '$.issue.pull_request') as is_pull_request,
450
  json_extract_string(payload, '$.issue.state_reason') as issue_state_reason,
451
+ -- Actor/assignee fields for assistant assignment
452
  json_extract_string(payload, '$.issue.user.login') as issue_creator,
453
  json_extract_string(payload, '$.issue.assignee.login') as issue_assignee,
454
  json_extract(payload, '$.issue.assignees') as issue_assignees,
 
481
  WHERE
482
  type IN ('IssuesEvent', 'IssueCommentEvent', 'PullRequestEvent', 'DiscussionEvent')
483
  AND (
484
+ -- Assistant-assigned issues: assistant is creator, assignee, or commenter
485
  (type = 'IssuesEvent' AND (
486
  json_extract_string(payload, '$.issue.user.login') IN ({identifier_list})
487
  OR json_extract_string(payload, '$.issue.assignee.login') IN ({identifier_list})
 
491
  )
492
  OR SPLIT_PART(json_extract_string(repo, '$.name'), '/', 1) IN ({tracked_orgs_list})
493
  ))
494
+ -- Issue comments: assistant is commenter OR tracked org
495
  OR (type = 'IssueCommentEvent' AND (
496
  json_extract_string(payload, '$.comment.user.login') IN ({identifier_list})
497
  OR SPLIT_PART(json_extract_string(repo, '$.name'), '/', 1) IN ({tracked_orgs_list})
498
  ))
499
+ -- PRs: assistant is creator OR tracked org (for wanted issue tracking)
500
  OR (type = 'PullRequestEvent' AND (
501
  json_extract_string(payload, '$.pull_request.user.login') IN ({identifier_list})
502
  OR SPLIT_PART(json_extract_string(repo, '$.name'), '/', 1) IN ({tracked_orgs_list})
503
  ))
504
+ -- Discussions: assistant is creator AND tracked org
505
  OR (type = 'DiscussionEvent'
506
  AND json_extract_string(payload, '$.discussion.user.login') IN ({identifier_list})
507
  AND SPLIT_PART(json_extract_string(repo, '$.name'), '/', 1) IN ({tracked_orgs_list})
 
522
  issue_events = [] # For wanted tracking
523
  pr_events = [] # For wanted tracking
524
  discussion_events = [] # For discussion tracking
525
+ agent_issue_events = [] # For assistant-assigned issues
526
 
527
  for row in all_results:
528
  event_type = row[0]
 
530
 
531
  if event_type in ('IssuesEvent', 'IssueCommentEvent'):
532
  if not is_pr: # It's an issue, not a PR
533
+ # Check if this is an assistant-assigned issue
534
  issue_creator = row[11]
535
  issue_assignee = row[12]
536
  issue_assignees_json = row[13]
 
564
  pass
565
 
566
  elif event_type == 'IssueCommentEvent':
567
+ # Check if commenter is an assistant
568
  if commenter in identifier_set:
569
  agent_identifier = commenter
570
 
 
584
  elif event_type == 'DiscussionEvent':
585
  discussion_events.append(row)
586
 
587
+ # Process assistant-assigned issues
588
  for row, agent_identifier in agent_issue_events:
589
  # Row indices: repo_url=2, issue_url=3, issue_created_at=6, issue_closed_at=7, issue_state_reason=10
590
  repo_url = row[2]
 
612
  except:
613
  continue
614
 
615
+ # Deduplicate: only add if we haven't seen this issue for this assistant
616
  if full_url in agent_issue_urls[agent_identifier]:
617
  continue
618
 
 
753
  except:
754
  continue
755
 
756
+ # Group by creator (assistant identifier)
757
+ # Only track discussions from our assistant identifiers
758
  if discussion_creator not in identifier_set:
759
  continue
760
 
 
776
  'state_reason': discussion_state_reason
777
  }
778
 
779
+ # Group by assistant
780
  if discussion_creator not in discussions_by_agent:
781
  discussions_by_agent[discussion_creator] = []
782
  discussions_by_agent[discussion_creator].append(discussion_meta)
783
 
784
+ print(f"✓ {len(agent_issue_events)} assistant issues, {len(issue_events)} wanted issues, {len(pr_events)} PRs, {len(discussion_events)} discussions")
785
 
786
  except Exception as e:
787
  print(f"\n ✗ Batch {batch_num} error: {str(e)}")
 
790
  # Move to next batch
791
  current_date = batch_end + timedelta(days=1)
792
 
793
+ # Post-processing: Filter issues and assign to assistants
794
  print(f"\n Post-processing {len(all_issues)} wanted issues...")
795
 
796
  wanted_open = []
 
803
  if not linked_prs:
804
  continue
805
 
806
+ # Check if any linked PR was merged AND created by an assistant
807
  resolved_by = None
808
  for pr_url in linked_prs:
809
  merged_at = pr_merged_at.get(pr_url)
 
862
  except:
863
  pass
864
 
865
+ print(f" ✓ Found {sum(len(issues) for issues in agent_issues.values())} assistant-assigned issues across {len(agent_issues)} assistants")
866
  print(f" ✓ Found {len(wanted_open)} long-standing open wanted issues")
867
+ print(f" ✓ Found {sum(len(issues) for issues in wanted_resolved.values())} resolved wanted issues across {len(wanted_resolved)} assistants")
868
+ print(f" ✓ Found {sum(len(discussions) for discussions in discussions_by_agent.values())} discussions across {len(discussions_by_agent)} assistants")
869
 
870
  return {
871
  'agent_issues': dict(agent_issues),
 
933
 
934
  def load_agents_from_hf():
935
  """
936
+ Load all assistant metadata JSON files from local git repository.
937
  ALWAYS syncs with remote first to ensure we have the latest bot data.
938
  """
939
  # MANDATORY: Sync with remote first to get latest bot data
940
+ print(f" Syncing bot_metadata repository to get latest assistants...")
941
  sync_agents_repo() # Will raise exception if sync fails
942
 
943
+ assistants = []
944
 
945
  # Scan local directory for JSON files
946
  if not os.path.exists(AGENTS_REPO_LOCAL_PATH):
 
948
 
949
  # Walk through the directory to find all JSON files
950
  files_processed = 0
951
+ print(f" Loading assistant metadata from {AGENTS_REPO_LOCAL_PATH}...")
952
 
953
  for root, dirs, files in os.walk(AGENTS_REPO_LOCAL_PATH):
954
  # Skip .git directory
 
966
  with open(file_path, 'r', encoding='utf-8') as f:
967
  agent_data = json.load(f)
968
 
969
+ # Only include active assistants
970
  if agent_data.get('status') != 'active':
971
  continue
972
 
 
974
  github_identifier = filename.replace('.json', '')
975
  agent_data['github_identifier'] = github_identifier
976
 
977
+ assistants.append(agent_data)
978
 
979
  except Exception as e:
980
  print(f" ⚠ Error loading {filename}: {str(e)}")
981
  continue
982
 
983
+ print(f" ✓ Loaded {len(assistants)} active assistants (from {files_processed} total files)")
984
+ return assistants
985
 
986
 
987
  def calculate_issue_stats_from_metadata(metadata_list):
 
1002
  }
1003
 
1004
 
1005
+ def calculate_monthly_metrics_by_agent(all_metadata_dict, assistants):
1006
+ """Calculate monthly metrics for all assistants for visualization."""
1007
+ identifier_to_name = {assistant.get('github_identifier'): assistant.get('name') for assistant in assistants if assistant.get('github_identifier')}
1008
 
1009
  if not all_metadata_dict:
1010
+ return {'assistants': [], 'months': [], 'data': {}}
1011
 
1012
  agent_month_data = defaultdict(lambda: defaultdict(list))
1013
 
 
1065
  agents_list = sorted(list(agent_month_data.keys()))
1066
 
1067
  return {
1068
+ 'assistants': agents_list,
1069
  'months': months,
1070
  'data': result_data
1071
  }
 
1086
  }
1087
 
1088
 
1089
+ def calculate_monthly_metrics_by_agent_discussions(all_discussions_dict, assistants):
1090
+ """Calculate monthly metrics for discussions for all assistants for visualization."""
1091
+ identifier_to_name = {assistant.get('github_identifier'): assistant.get('name') for assistant in assistants if assistant.get('github_identifier')}
1092
 
1093
  if not all_discussions_dict:
1094
+ return {'assistants': [], 'months': [], 'data': {}}
1095
 
1096
  agent_month_data = defaultdict(lambda: defaultdict(list))
1097
 
 
1145
  agents_list = sorted(list(agent_month_data.keys()))
1146
 
1147
  return {
1148
+ 'assistants': agents_list,
1149
  'months': months,
1150
  'data': result_data
1151
  }
1152
 
1153
 
1154
+ def construct_leaderboard_from_metadata(all_metadata_dict, assistants, wanted_resolved_dict=None, discussions_dict=None):
1155
  """Construct leaderboard from in-memory issue metadata and discussion metadata.
1156
 
1157
  Args:
1158
+ all_metadata_dict: Dictionary mapping assistant ID to list of issue metadata (assistant-assigned issues)
1159
+ assistants: List of assistant metadata
1160
+ wanted_resolved_dict: Optional dictionary mapping assistant ID to list of resolved wanted issues
1161
+ discussions_dict: Optional dictionary mapping assistant ID to list of discussion metadata
1162
  """
1163
+ if not assistants:
1164
+ print("Error: No assistants found")
1165
  return {}
1166
 
1167
  if wanted_resolved_dict is None:
 
1172
 
1173
  cache_dict = {}
1174
 
1175
+ for assistant in assistants:
1176
+ identifier = assistant.get('github_identifier')
1177
+ agent_name = assistant.get('name', 'Unknown')
1178
 
1179
  bot_metadata = all_metadata_dict.get(identifier, [])
1180
  stats = calculate_issue_stats_from_metadata(bot_metadata)
 
1188
 
1189
  cache_dict[identifier] = {
1190
  'name': agent_name,
1191
+ 'website': assistant.get('website', 'N/A'),
1192
  'github_identifier': identifier,
1193
  **stats,
1194
  'resolved_wanted_issues': resolved_wanted,
 
1211
  wanted_issues = []
1212
 
1213
  if discussion_monthly_metrics is None:
1214
+ discussion_monthly_metrics = {'assistants': [], 'months': [], 'data': {}}
1215
 
1216
  combined_data = {
1217
  'metadata': {
 
1255
 
1256
  def mine_all_agents():
1257
  """
1258
+ Mine issue metadata for all assistants using STREAMING batch processing.
1259
  Downloads GHArchive data, then uses BATCH-based DuckDB queries.
1260
  """
1261
  print(f"\n[1/4] Downloading GHArchive data...")
 
1263
  if not download_all_gharchive_data():
1264
  print("Warning: Download had errors, continuing with available data...")
1265
 
1266
+ print(f"\n[2/4] Loading assistant metadata...")
1267
 
1268
+ assistants = load_agents_from_hf()
1269
+ if not assistants:
1270
+ print("Error: No assistants found")
1271
  return
1272
 
1273
+ identifiers = [assistant['github_identifier'] for assistant in assistants if assistant.get('github_identifier')]
1274
  if not identifiers:
1275
+ print("Error: No valid assistant identifiers found")
1276
  return
1277
 
1278
+ print(f"\n[3/4] Mining issue metadata ({len(identifiers)} assistants, {LEADERBOARD_TIME_FRAME_DAYS} days)...")
1279
 
1280
  try:
1281
  conn = get_duckdb_connection()
 
1309
 
1310
  try:
1311
  leaderboard_dict = construct_leaderboard_from_metadata(
1312
+ agent_issues, assistants, wanted_resolved, agent_discussions
1313
  )
1314
+ monthly_metrics = calculate_monthly_metrics_by_agent(agent_issues, assistants)
1315
  discussion_monthly_metrics = calculate_monthly_metrics_by_agent_discussions(
1316
+ agent_discussions, assistants
1317
  )
1318
  save_leaderboard_data_to_hf(
1319
  leaderboard_dict, monthly_metrics, wanted_open, discussion_monthly_metrics
 
1350
  mine_all_agents,
1351
  trigger=trigger,
1352
  id='mine_all_agents',
1353
+ name='Mine GHArchive data for all assistants',
1354
  replace_existing=True
1355
  )
1356