]*>([\d.]+)", str(score_str))
if match:
return float(match.group(1))
# Otherwise try to convert directly
try:
return float(score_str)
except Exception:
return None
# Sort by Full Set Score descending (or Text-Only Score minus 3% if Full Set Score is None or "-")
if "Full Set Score" in merged_df.columns and "Text-Only Score" in merged_df.columns:
merged_df["_full_score"] = merged_df["Full Set Score"].apply(extract_score_for_sorting)
merged_df["_text_score"] = merged_df["Text-Only Score"].apply(extract_score_for_sorting)
# Calculate sort score: use Full Set Score if available, otherwise Text-Only Score * 0.97
def calculate_sort_score(row):
full_score = row["_full_score"]
text_score = row["_text_score"]
if full_score is not None:
return full_score
elif text_score is not None:
return text_score * 0.97 # Minus 3%
else:
return 0
merged_df["_sort_score"] = merged_df.apply(calculate_sort_score, axis=1)
merged_df = merged_df.sort_values("_sort_score", ascending=False, na_position="last")
merged_df = merged_df.drop(columns=["_sort_score", "_full_score", "_text_score"])
elif "Full Set Score" in merged_df.columns:
merged_df["_sort_score"] = merged_df["Full Set Score"].apply(extract_score_for_sorting)
merged_df = merged_df.sort_values("_sort_score", ascending=False, na_position="last")
merged_df = merged_df.drop(columns=["_sort_score"])
# Format scores that aren't already links
def format_score_with_ref(score_str):
"""Format score, ensuring it has 2 decimal places if it's a link"""
if not score_str:
return ""
# If already an HTML link with tooltip and bold, format the number inside
tooltip_pattern = (
r']*>([\d.]+)' r'([^<]+)'
)
match = re.match(tooltip_pattern, str(score_str))
if match:
url = match.group(1)
score_val = float(match.group(2))
tooltip_text = match.group(3)
return (
f''
f'{score_val:.2f}{tooltip_text}'
)
# If already an HTML link with tooltip (without bold), format the number inside
tooltip_pattern = r']*>([\d.]+)' r'([^<]+)'
match = re.match(tooltip_pattern, str(score_str))
if match:
url = match.group(1)
score_val = float(match.group(2))
tooltip_text = match.group(3)
return (
f''
f'{score_val:.2f}{tooltip_text}'
)
# If it's a simple link without tooltip wrapper, add tooltip wrapper
match = re.match(r']*>([\d.]+)', str(score_str))
if match:
url = match.group(1)
score_val = float(match.group(2))
return (
f''
f'{score_val:.2f}{url}'
)
# If it's already bold but not a link
match = re.match(r"([\d.]+)", str(score_str))
if match:
score_val = float(match.group(1))
return f"{score_val:.2f}"
# Otherwise format as plain number with bold
try:
score_val = float(score_str)
return f"{score_val:.2f}"
except Exception:
# If it's already a string with special characters (like "-"), keep it as is but make bold
return f"{score_str}"
merged_df["Full Set Score"] = merged_df["Full Set Score"].apply(format_score_with_ref)
if "Text-Only Score" in merged_df.columns:
def format_score_with_ref(score_str):
"""Format score, ensuring it has 2 decimal places if it's a link"""
if not score_str:
return ""
# If already an HTML link with tooltip, format the number inside
tooltip_pattern = r']*>([\d.]+)' r'([^<]+)'
match = re.match(tooltip_pattern, str(score_str))
if match:
url = match.group(1)
score_val = float(match.group(2))
tooltip_text = match.group(3)
return (
f''
f'{score_val:.2f}{tooltip_text}'
)
# If it's a simple link without tooltip wrapper, add tooltip wrapper
match = re.match(r']*>([\d.]+)', str(score_str))
if match:
url = match.group(1)
score_val = float(match.group(2))
return (
f''
f'{score_val:.2f}{url}'
)
# Otherwise format as plain number
try:
score_val = float(score_str)
return f"{score_val:.2f}"
except Exception:
return str(score_str)
merged_df["Text-Only Score"] = merged_df["Text-Only Score"].apply(format_score_with_ref)
# Ensure Organization, Open Source, and Publish Date columns exist
if "Organization" not in merged_df.columns:
merged_df["Organization"] = ""
if "Open Source" not in merged_df.columns:
merged_df["Open Source"] = ""
if "Publish Date" not in merged_df.columns:
merged_df["Publish Date"] = ""
# Rename Model column to "Agent / Model"
if "Model" in merged_df.columns:
merged_df = merged_df.rename(columns={"Model": "Agent / Model"})
# Reorder columns: Agent / Model, Organization, Open Source, Publish Date, Full Set Score, Text-Only Score
column_order = ["Agent / Model", "Organization", "Open Source", "Publish Date", "Full Set Score", "Text-Only Score"]
existing_columns = [col for col in column_order if col in merged_df.columns]
other_columns = [col for col in merged_df.columns if col not in column_order]
merged_df = merged_df[existing_columns + other_columns]
return merged_df.reset_index(drop=True)
def markdown_to_html(text: str) -> str:
"""Convert markdown text to HTML while preserving HTML badges"""
import re
lines = text.split("\n")
html_lines = []
in_list = False
in_paragraph = False
paragraph_lines = []
def process_inline_markdown(line: str) -> str:
"""Process inline markdown (bold, links) in a line"""
# Convert markdown links [text](url) to HTML links
line = re.sub(r"\[([^\]]+)\]\(([^\)]+)\)", r'\1', line)
# Convert markdown bold **text** to HTML bold
line = re.sub(r"\*\*([^\*]+)\*\*", r"\1", line)
return line
for line in lines:
stripped = line.strip()
# Handle headers
if stripped.startswith("#### "):
if in_list:
html_lines.append("")
in_list = False
if in_paragraph:
html_lines.append("" + " ".join(paragraph_lines) + "
")
paragraph_lines = []
in_paragraph = False
header_text = process_inline_markdown(stripped[5:])
html_lines.append(f"{header_text}
")
elif stripped.startswith("### "):
if in_list:
html_lines.append("")
in_list = False
if in_paragraph:
html_lines.append("" + " ".join(paragraph_lines) + "
")
paragraph_lines = []
in_paragraph = False
header_text = process_inline_markdown(stripped[4:])
html_lines.append(f"{header_text}
")
# Handle list items
elif stripped.startswith("- "):
if in_paragraph:
html_lines.append("" + " ".join(paragraph_lines) + "
")
paragraph_lines = []
in_paragraph = False
if not in_list:
html_lines.append("")
in_list = True
list_item = process_inline_markdown(stripped[2:])
html_lines.append(f"- {list_item}
")
# Handle empty lines
elif not stripped:
if in_list:
html_lines.append("
")
in_list = False
if in_paragraph:
html_lines.append("" + " ".join(paragraph_lines) + "
")
paragraph_lines = []
in_paragraph = False
html_lines.append("
")
# Handle regular paragraphs
else:
if in_list:
html_lines.append("")
in_list = False
processed_line = process_inline_markdown(stripped)
paragraph_lines.append(processed_line)
in_paragraph = True
# Close any open structures
if in_list:
html_lines.append("")
if in_paragraph:
html_lines.append("" + " ".join(paragraph_lines) + "
")
return "\n".join(html_lines)
def create_motivation_section():
"""Create the motivation section content as HTML"""
motivation = f"""### About Humanity's Last Exam (HLE)
[Humanity's Last Exam (HLE)](https://agi.safe.ai/) is a rigorous, multi-modal AI benchmark created by the Center for AI Safety in collaboration with Scale AI,
designed to push large language models beyond saturated tests by evaluating reasoning and expert-level knowledge across thousands of
challenging questions spanning mathematics, natural sciences, and the humanities.
### Why another leaderboard?
While leaderboards existed for HLE, they fall short in several ways, leading to widespread confusion about true state-of-the-art results.
In fact, if you ask ChatGPT, Gemini, and Claude "What's the SOTA for Humanity's Last Exam" today, they will all get the answer wrong.
- **Tool Exclusion**: The [official leaderboard by Scale AI](https://scale.com/leaderboard/humanitys_last_exam) and many other leaderboards focus on models without tool use.
- **Lack of separation**: Scores for the full HLE benchmark and the text-only subset are often mixed together, leading to unfair comparisons - score on full set is generally lower than the text only subset.
- **Data Contamination**: Since copies of HLE, blogs and papers discussing HLE have been indexed by search engines, scores might be artificially inflated for agents without filtering. In this leaderboard, we add a{VERIFIED_BADGE} badge to indicate that some form of filtering is mentioned for the agent or its previous versions.
"""
return markdown_to_html(motivation)
def build_leaderboard():
"""Build the Gradio interface"""
# Get data (from files or defaults)
full_set_data, text_only_data = get_data()
# Create separate tables for Full Set and Text-Only
full_set_df = format_leaderboard_table(full_set_data, "Full Set Score")
text_only_df = format_leaderboard_table(text_only_data, "Text-Only Score")
custom_css = """
.gradio-container {
max-width: 1200px;
width: 100%;
margin: 0 auto;
padding: 0 10px;
overflow: visible !important;
}
.gradio-row {
justify-content: center;
overflow: visible !important;
}
.gradio-column {
text-align: left;
max-width: 100%;
overflow: visible !important;
}
h1, h2, h3 {
text-align: center !important;
}
.markdown-text h1, .markdown-text h2, .markdown-text h3 {
text-align: center !important;
}
hr {
border-color: #95d5b2 !important;
border-width: 2px !important;
}
.markdown-text, .markdown-text p, .markdown-text ul, .markdown-text ol, .markdown-text li {
text-align: left !important;
}
/* Style verified badge in markdown - restore badge styling for all spans if inline styles are stripped */
.markdown-text span {
display: inline-block !important;
width: 12px !important;
height: 12px !important;
background-color: #00bfa5 !important;
border-radius: 50% !important;
color: white !important;
text-align: center !important;
line-height: 12px !important;
font-size: 8px !important;
font-weight: bold !important;
margin-left: 4px !important;
vertical-align: middle !important;
}
table {
border: 2px solid #95d5b2 !important;
border-collapse: collapse;
margin: 0 auto;
}
table th {
background-color: #95d5b2 !important;
color: #1a1a1a !important;
text-align: center;
}
table td {
text-align: center;
}
table tr:hover {
background-color: rgba(149, 213, 178, 0.1) !important;
}
table a {
color: inherit !important;
text-decoration: none !important;
}
table a:hover {
color: #00bfa5 !important;
}
/* Make Agent / Model column bold */
table td:first-child, table th:first-child {
font-weight: bold !important;
}
/* Make scores green */
table td strong {
color: #00bfa5 !important;
}
table td a strong {
color: #00bfa5 !important;
}
/* Custom tooltip styling */
.tooltip {
position: relative;
display: inline-block;
}
.tooltip .tooltiptext {
visibility: hidden;
opacity: 0;
background-color: rgba(60, 60, 60, 0.95);
color: #ffffff;
text-align: center;
border-radius: 0;
padding: 14px 18px;
position: absolute;
z-index: 9999;
bottom: 125%;
left: 50%;
transform: translateX(-50%);
font-size: 13px;
box-shadow: 0 6px 16px rgba(0, 0, 0, 0.3);
transition: opacity 0.05s ease-in, visibility 0.05s;
pointer-events: none;
margin-bottom: 15px;
word-wrap: break-word;
white-space: nowrap;
line-height: 1.5;
border: none;
}
.tooltip .tooltiptext::after {
content: "";
position: absolute;
top: 100%;
left: 50%;
margin-left: -6px;
border-width: 6px;
border-style: solid;
border-color: rgba(60, 60, 60, 0.95) transparent transparent transparent;
}
.tooltip:hover .tooltiptext {
visibility: visible;
opacity: 1;
transition: opacity 0.05s ease-in, visibility 0s;
}
/* Mobile responsive styles */
@media screen and (max-width: 768px) {
.gradio-container {
max-width: 100%;
padding: 0 5px;
}
table {
font-size: 12px;
}
table td, table th {
padding: 6px 4px !important;
font-size: 11px;
}
h1 {
font-size: 24px !important;
}
h2 {
font-size: 20px !important;
}
h3 {
font-size: 18px !important;
}
.markdown-text {
font-size: 14px !important;
}
.tooltip .tooltiptext {
font-size: 11px;
padding: 10px 12px;
white-space: normal;
}
}
@media screen and (max-width: 480px) {
.gradio-container {
padding: 0 3px;
}
table {
font-size: 10px;
}
table td, table th {
padding: 4px 2px !important;
font-size: 10px;
}
h1 {
font-size: 20px !important;
}
h2 {
font-size: 18px !important;
}
h3 {
font-size: 16px !important;
}
.markdown-text {
font-size: 13px !important;
}
}
"""
with gr.Blocks(title="HLE Leaderboard for Agents with Tools", theme="JohnSmith9982/small_and_pretty", css=custom_css) as demo:
gr.Markdown(
"""
# 🏆 Humanity's Last Exam Leaderboard for Agents with Tools
"""
)
gr.Markdown("---")
# Motivation Section
with gr.Row():
with gr.Column():
motivation_text = create_motivation_section()
gr.HTML(motivation_text)
gr.Markdown("---")
gr.Markdown("
")
# Full Set Leaderboard
gr.Markdown("## 📊 Full Set Leaderboard")
gr.Markdown(
"""
Results on the complete set of tasks (2500 examples).
"""
)
# Convert DataFrame to HTML with clickable links
full_set_html = dataframe_to_html(full_set_df)
# Display as HTML to support clickable links
gr.HTML(value=full_set_html)
# Text-Only Leaderboard
gr.Markdown("
")
gr.Markdown("## 📊 Text-Only Leaderboard")
gr.Markdown(
"""
Results on the text-only subset of tasks (2158 examples).
"""
)
# Convert DataFrame to HTML with clickable links
text_only_html = dataframe_to_html(text_only_df)
# Display as HTML to support clickable links
gr.HTML(value=text_only_html)
# Add JavaScript for tooltip positioning (injected after page load)
gr.HTML(
value="""""",
visible=False,
)
# Footer
gr.Markdown("
")
notes_text = f"""#### Notes
- All models and agents listed have tool use capabilities.
- All numbers are based on the official report of each agent.
- Missing{VERIFIED_BADGE} badge does not necessarily mean the agent applied no filtering, it just means that no such mention is found.
- Having{VERIFIED_BADGE} badge does not necessarily mean the agent applied perfect filtering. E.g., only blocking huggingface URLs won't be sufficient.
- We exclude scores reported on non-official subsets as the results are not comparable.
- Please contact us at hle@zoom.us if you find any errors or want to add a model or agent to the leaderboard.
- Last updated: Jan 8, 2026.
"""
gr.HTML(markdown_to_html(notes_text))
return demo
if __name__ == "__main__":
demo = build_leaderboard()
demo.launch(share=False, server_name="0.0.0.0", server_port=7860)